{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "**Chapter 15 – Autoencoders**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "_This notebook contains all the sample code and solutions to the exercices in chapter 15._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# To support both python 2 and python 3\n",
    "from __future__ import division, print_function, unicode_literals\n",
    "\n",
    "# Common imports\n",
    "import numpy as np\n",
    "import numpy.random as rnd\n",
    "import os\n",
    "import sys\n",
    "\n",
    "# to make this notebook's output stable across runs\n",
    "rnd.seed(42)\n",
    "\n",
    "# To plot pretty figures\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['axes.labelsize'] = 14\n",
    "plt.rcParams['xtick.labelsize'] = 12\n",
    "plt.rcParams['ytick.labelsize'] = 12\n",
    "\n",
    "# Where to save the figures\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"autoencoders\"\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True):\n",
    "    path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n",
    "    print(\"Saving figure\", fig_id)\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format='png', dpi=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "A couple utility functions to plot grayscale 28x28 image:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def plot_image(image, shape=[28, 28]):\n",
    "    plt.imshow(image.reshape(shape), cmap=\"Greys\", interpolation=\"nearest\")\n",
    "    plt.axis(\"off\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def plot_multiple_images(images, n_rows, n_cols, pad=2):\n",
    "    images = images - images.min()  # make the minimum == 0, so the padding looks white\n",
    "    w,h = images.shape[1:]\n",
    "    image = np.zeros(((w+pad)*n_rows+pad, (h+pad)*n_cols+pad))\n",
    "    for y in range(n_rows):\n",
    "        for x in range(n_cols):\n",
    "            image[(y*(h+pad)+pad):(y*(h+pad)+pad+h),(x*(w+pad)+pad):(x*(w+pad)+pad+w)] = images[y*n_cols+x]\n",
    "    plt.imshow(image, cmap=\"Greys\", interpolation=\"nearest\")\n",
    "    plt.axis(\"off\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# PCA with a linear Autoencoder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Build 3D dataset:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "rnd.seed(4)\n",
    "m = 100\n",
    "w1, w2 = 0.1, 0.3\n",
    "noise = 0.1\n",
    "\n",
    "angles = rnd.rand(m) * 3 * np.pi / 2 - 0.5\n",
    "X_train = np.empty((m, 3))\n",
    "X_train[:, 0] = np.cos(angles) + np.sin(angles)/2 + noise * rnd.randn(m) / 2\n",
    "X_train[:, 1] = np.sin(angles) * 0.7 + noise * rnd.randn(m) / 2\n",
    "X_train[:, 2] = X_train[:, 0] * w1 + X_train[:, 1] * w2 + noise * rnd.randn(m)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Normalize the data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler = StandardScaler()\n",
    "X_train = scaler.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Going to need TensorFlow..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now let's build the Autoencoder..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Note: instead of using the `fully_connected()` function from the `tensorflow.contrib.layers` module (as in the book), we now use the `dense()` function from the `tf.layers` module, which did not exist when this chapter was written. This is preferable because anything in contrib may change or be deleted without notice, while `tf.layers` is part of the official API. As you will see, the code is mostly the same.\n",
    "\n",
    "The main differences relevant to this chapter are:\n",
    "* the `scope` parameter was renamed to `name`, and the `_fn` suffix was removed in all the parameters that had it (for example the `activation_fn` parameter was renamed to `activation`).\n",
    "* the `weights` parameter was renamed to `kernel` and the weights variable is now named `\"kernel\"` rather than `\"weights\"`,\n",
    "* the bias variable is now named `\"bias\"` rather than `\"biases\"`,\n",
    "* the default activation is `None` instead of `tf.nn.relu`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "n_inputs = 3\n",
    "n_hidden = 2  # codings\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "hidden = tf.layers.dense(X, n_hidden)\n",
    "outputs = tf.layers.dense(hidden, n_outputs)\n",
    "\n",
    "mse = tf.reduce_mean(tf.square(outputs - X))\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "training_op = optimizer.minimize(mse)\n",
    "\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "n_iterations = 10000\n",
    "codings = hidden\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for iteration in range(n_iterations):\n",
    "        training_op.run(feed_dict={X: X_train})\n",
    "    codings_val = codings.eval(feed_dict={X: X_train})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure linear_autoencoder_pca_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARgAAADQCAYAAADcQn7hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAG5VJREFUeJztnX2wXGV9xz/f3ERS5EZTEKe8JCRESci1AdTEiUICFoIU\n0dIXSi3SYrUifTFFoSMp4cVKsdORVhFQQYqAQFNFxTi0dExCyW3i1GlsgMAgeZFcBB1DsoBUEn79\n4znHnGx2757dPWfPObu/z8zO7tl9zrPPnrv7u8/ze/k+MjMcx3HyYELRA3Acp39xA+M4Tm64gXEc\nJzfcwDiOkxtuYBzHyQ03MI7j5IYbGMdxcqNQAyPpVZK+JGmLpJ2S/lvS6U3ani9pt6RdkmrR/Um9\nHrPjOOmZWIL33wacaGY/kvSbwD2SRsxsW4P2a83MjYrjVIRCDYyZvQhclTj+tqTNwJsJhsdxnApT\nKh+MpNcDbwAebtLkeEnPStokaZmkUo3fcZx9KXqJ9EskTQRuB241s8cbNFkNjJjZVklzgXuAl4Fr\nezhMx3HaQGUodpQk4KvAQcB7zGxPinPOAT5mZm9t8FrxH8px+hwzU6s2ZVli3AwcApydxrgkaPoB\nzSy32/Lly73/Avvvh89Q9f7TUriBkXQjMBs4y8x+MU670yUdGj2eDSwD7u3NKB3H6YSi82CmAR8C\njgOeSeS3nCvpyOj4iKj5O4EfSKoB9wErgGuKGbnjOGkoOky9jfGN3HCi7ceBj+c+qBQsXrzY+y+w\n/168h/efDaVw8maNJOvHz+U4ZUESViEnr+M4fYgbGMdxcsMNjOM4ueEGxnEcAGo1GB0N91nhBsZx\nHGo1OPHEcDvhBBgby6ZfNzCO47BxY7jt2QNPPAGLFmUzkyk60S614FTUfqmkpyXtiM6b1MvxOk6/\nMjICM2bsPd66FR5upmnQBkXPYJKCU68BLicITk2rbyhpCXAJcDJwFHA0cGXvhuo4/cvwMKxeDbNm\nwaRJcOyxMHdu9/2WLtFO0gbgCjP7et3zdwCbzWxZdHwKcIeZ/VqDPjzRznE6oFYLM5e5c4PRaUYl\nE+1aCE7NBTYkjjcAh0qa2ouxOc4gMDwMb3vb+MalHaokOHUQsDNxvJMg1zAM7KhvfMUVV/zy8eLF\ni0tTm+E4VWTVqlWsWrWq7fNKsURKIzgl6X+AT5rZiuj4V4GfAIeY2Y66tr5EcgaeWi1EhkZGspuR\nxFRtiZRGcOphYF7i+DjgmXrj4jjO3ryWk04K91kmz7VD4QYmreAUcBvwAUlzIr/LZcCXezFGx6ka\nGzcGZ+3u3fDII9mEnDuh6DyYVoJTu2LBKTO7H/g08F1gc3S7oqChO06pGRkJkaAsQ86dUAofTNa4\nD8Zx0oecOyGtD8YNjOM4bVM1J6/jOE3otMo5j+rodnED4zglptNoUPK8hQvhgQeKMTRuYBynxHQa\nDUqet3EjnH56MeFqNzCOU2I6jQbF502McvX37CkmXO1OXscpOZ1Gg2o1WL8ePvpReOyxYKAefDCb\niFJlokiSLgL+CHgTcKeZXdCk3fmEjN8XCTVIBpxpZmsatHUD4zgReYSr0xqYMhQ7bgeuBpYAv9Ki\n7VozOyn/ITlOeWm3xiiukC6Cwn0wZnavmX0T+FnRY3GcslOWGqO0FG5g2uR4Sc9K2iRpmaSqjd9x\nuqIsNUZpqdIPdDUwYmaHAr8NnEtJ9qp2nF5RlhqjtJTBB5MKM9uSePywpKuAjwHXNmrvglNOPzI8\nHCJB69dDL+MYlRacApB0NXB4syhSg/bnAB83s7c0eM2jSE7p6VQQKvbDxJGhrELP7VCZWiRJQ5Im\nA0PAREkHSBpq0O50SYdGj2cDy4B7eztax8mGbpy19X6Yu+4qr7O3cANDMBQvApcC74seXxbpwdRi\nPRjgncAPJNWA+4AVwDVFDNhxuqUbZ23SDzM0BBdeWN6IUmmWSFniSySn7MQzmEce6SzDtlYLM5cL\nLwxlAJMmwZo1vct3qUwmbx64gXGqQLcZtt0aqW5wA9OHn8vpT2Jn7/TpYcvWkZHwfBoHcJ6qdePh\nBqYPP5fTf8SzkI0bwzJn926YPTu8tmnT3igR7G+Eeh05SuIGpg8/V9XIc1+eMtPO5x4dDZGk3bv3\nPjc0BFJ4btIk+M534OKL9zVCRYWnYyoTpnb6k6rVzGRFu587qdsyeXIwIHPmwBvfGJ475piQUPfw\nw8GZ+9JL1SkTADcwTk5UrWYmK9r93HFm7oMPwg9/GCJB998PEybszdSNSwKSRqgKZQLgSyQnJ4qM\ncBRJFp87uWyKw89z5wZjNW0abNvWe6duPZXxwaQVnIraLgUuASYD/wpcaGYvN2jnBqYEFBXhKJoq\nh5/TUiUD817gFSLBqXEU7ZYAtwInA08TygRGzewTDdq6gXEqSTJkXYaZSjMq4+RtQ3Dq/cDNZrbJ\nzHYSVPD+OPcBOr+kfp+dMuy7008kHcRnnFFe49IOhRuYNpgLbEgcbwAOlTS1oPEMFLVa2F8n3mdn\nbGwwo0R5sm5d/znGK6MHAxwE7Ewc7ySIfw8DO+obux5MtqxbF6buEO5XrNj/x1CU7ms/UKvB0qV7\n82GOOaZcUaJO9WCqZGCeB6YkjqcQdhZo+L8zaWCc7JkxI/wAYkdkmX4MVWTjxpC5CyHR7rrryrU8\nqv8nfeWVV6Y6r0pLpIeBeYnj44BnzGy/2YuTPQsWhKSwoaFwv3hxiG6sWVPOKEfVSEowjIzA/PlF\njygbyhBFGgImAZcDRwAfBHab2Z66dkuALxN0YX5M0IP5LzO7rEGfHkXKgUENO/eKKl3fKoWplwPL\nCcudmCsJxuQRYI6ZPRW1/Sjw14Q8mBV4HozjFEJlDEweuIFpzaAWIhZNv1z3yuTBOL1nUAsRiyLO\nFxrE0L4bmAFkUAsRiyBpzBctCtd+kK67G5gBpGqbd1WZpDHfujWE9wfpursPZkCpUsSiStT7WOoL\nF1euLHeNUVrcyduHn6sT+sWpWAWabYjWj8bcnbyOO3N7TDPf1vBwKKPoF+PSDqkNjKQDJa2Q9Fyk\n4YKk8ySd0M0AJE2V9HVJz0vaLOncJu2WS/qFpF3Rhmy7JB3VzXv3M7UafPWrzZ25XgmdPdOnB9U5\nCBnP06YVO54y0M4M5lJgLUEi4XBJdwIHm9n3uxzD54GXgNcBfwjcIGlOk7Z3mdkUMxuO7rd0+d59\nSTxz+chHwhe+3qnoM5vuGRuDL3wh3Mds3QovR2mfu3cHX8ug006x42fM7DkASduAJWZ2XXR8NjAd\nWAA8amapKqEkHQicDRxrZj8HHpL0TeA8YD8hKScd8VR9z56gTn/DDXDOOXun6I2m8l4JnZ6xMTj6\n6CDAPXly0NI97LDg5xoZ8QLQJKlnMAnjcirwdjP7VHQ8E3itmX2GIH15QbNlTgPeSKg7+mHiuQ0E\n7ZdGvFvSTyX9r6QPpx37oJEMQ8+du69xqX/dfwjtc999wbhAuF+5MjyOBby9AHQvbUWRIsMxZGa3\nJ547C/icmU2Ljv+FUOX8Zyn6ewdwj5kdlnjuT4A/MLNT6trOBp4DngHeRtDkXWpmdzfod+CjSK0i\nF/0Y2egVzWYwg0TaKFLqJZKk84Gfmdm3Es9dA/wNcEai6eHA6pTd1mu8EB3v5xUws02Jw1FJ/wj8\nDrCfgYHBE5yqD0fHkYtmtHrdac5hhwWjsnJlkLYcBOPSqeBUqhmMpN8i+EW2AI8Du6LjG83sG4l2\nxwN3AcdFPpVW/R5I0OKdGy+TJP0zsL2RmHfduZcA883sdxq8NlAzmGb5F46TF5nlwUh6PXCMmZ0N\nfBa4EPh74Ct1xuUA4ArgtDTGBcDMXgS+BlwVhcHfDpwFfKXBOM6S9Nro8XzgLwg7Cww8XlvklJXM\nMnklXQ3cZGZPSZplZk+kPG8qcAtwKvBT4FIzuzvyz6w0sylRuzuB04BXAU8B15vZ9U36HMgZTJn3\n0XH6i56WCkj6EPC/wJMEv867zezGrjvufDwDZ2DWrQsh6fnz3bg4+ZO5k3ecN1pISJZTdDPgd7vt\n10lHI/+L0z1ew5UNXdcimdlaM5toZkNmNiG6/1oWg3Na4/6X7KnPdB4b87KKTvFix4rjSXPZU2+0\nFy3ysopOcQNTUeJiRfDs0axJGu3p02HzZp8hdorrwVQQz3vJnzjTedq0kEznEbp9ccGpPvxcMaOj\nYcq+e3f4L7tmjWflpqFTx62XVexPZQSn0urBRG2vjYodfyLp2l6Os0y436V9upGoGGTBqG4p3MCQ\nUg9G0p8SsnzfBPw6cGaUf9M3pBWB8qrd9vFoWzEUamASejDLzOznZvYQEOvB1PN+4B/M7Gkzexr4\nB4I8RF+Q9j9s0rnr/1WbU2+sfdZXDEXPYNrRg5kbvdaqXSVJ8x/WlejS0eg6+ayvGIo2MAcBO+ue\n2wk0+vPXt90ZPdcX1P+HnTZt/+WST/PT4eLb5aHrUoEuSa0H06DtlOi5hlRNDyb+D5sMjdaHoWMj\n5JKM4+PXKXty1YPJi3b0YCQ9BNxiZjdHxxcAf2JmCxv0W/ow9Xgh0/HC0B4yTYdfp3ypTB5MJMNg\nwAeB44H7gIVm9mhduz8laMCcGj31b8A/mtkXG/RZagPTKlHO5Rda48WIxVKZPBjgIuBA4FngDuDD\nZvaopHdI2hU3MrObgG8RZCF+AHyrkXGpAq18Ke6QHB93dleHwmcweVCmGUys1QJhNrJ1a6hv8fTz\n1jTa53njRnjhBXjXuzyTuUgqs0TKg7IYmFoNFi4MPwqAAw4IP4qRkf7ZBD0v4lnKxo0wYwZ8+9vw\ne78XZnuzZ4c2jz3mBroo3MCU4HONjoYfyZ49+z7v/3VbU3/tjjgCnn46HE+aBN/5Drz61W6gi6JK\nPpi+ZWQE5iSKHg44IGzl6qHT5sQZuAcfHPZ3jvnxj8NMJs4Tmj/fc1qqgM9gcqZWg/Xrw+M5c3xZ\nNB7J6NpRR4W9h+I/46xZsHq1X7+y4EukPvxc/UjSCQ77Om+nT4ctW4KxWb16MDY4qwpuYPrwc/Ub\nY2NhxvLkk+H4mGPg5ZfhRz8KyyB3hJcXNzB9+LmqTjLsDHDCCfBE3e5ZEybAzJk+Yyk7lXDytik2\ntVzSLyTtklSL7o/q3WjTU6vBAw+EmyeBBeqT49atC8ufel55JeQKbdvW8yE6OVB0FCmV2FSCu8xs\nipkNR/dbejHIdohzX049NdwWLtxrZNIKSvUj9dnL0t7q8Zkz4e67w8zG9Vr6i8KWSFGh4w7g2ESh\n423AU402vpe0HDjazN6fou/ClkjJQkUIodb//M/wg+lHoe542XPwwbBqFZx5ZuOlTaP6KggRNjNY\nsCAce4FiNUi7RMLMCrkBxwEv1D13MfCNJu2XEwzSTwn1SB8ep28ril27zEZGzMLPJjzetcts7Vqz\niRPDc5MmmY2OFjbEzIg/64QJZlL4bJMnm23f3rz96Gi4j4/nzQvXZd68vc875Sf6jbX8nRepB9OO\n2BTA3cBNwDPA24B/lbTDzO5u1LgoPZjhYVi7dm/uS7xXdD9qlKxbt7cMIuall+Czn4VPfGL/WUgs\n+BTTqOjTs5vLSen0YCR9F1hEkGKo5yGC9MJDZvbqxDl/BSwys/ek6P9S4C1mtt8+2GWNIvWbRskD\nDwQ/UxIpLAvTLANdlqK6FB5FMrOTbe9e1fW3k4DHgSFJRydOmwekFYI0oPUasET0m2TjggVhZjZx\nYshhWbo0GJe0kp4uS9H/FK1ol0psKmp7FrDGzJ6TNB/4GvDXZnZ7g7alnMGkpWgxpXbePzkrA5+R\nDAqVSLSTNBW4haBS91Pg0tinIukdwEozmxId3wmcBrwKeAq43syub9JvLgamFz/8oreF7fb9+20Z\n6DSmEgYmL/IwMJ3+8No1Smm3hc3L2HW6LW3Rsy6ntxTug+k3OtkypBNpxzQbhOUpGdnJBmUuYek0\nww1MSjr54XVilNI4Ptvtt50M4k4cr75fk9OUNMkyVbuRUaJdnCCXTAxLJoqlOX/evJBYl2Ui2Xj9\nNhpzMplt+/Z9X897PE5/QspEO/fBNCErZ2teTs9G/TYa88aN+/pUYo2VrB3I7twdLNwH0yXjTfvb\nXXLETtJm58T9jY2132/yx9xozMml3fTpsHlzPkuZfsvxcTIizTSnajcyWCI1m/Z3Uj8z3jnxa0ND\noY6nm7qc8cY8OhqWR76UcbKAlEukIo3ARcD3CHINt6RovxR4mlDw+CVg0jhtM7mIjXwunRQtjndO\n8rX41k0xZCs/Ubt+JMdpRFoDU6Rcw3uBV4AlwK+Y2QXjtF0C3AqcTDAy9wKj1kDWIWpveX2uTupn\nxjsn6TeZODFsy+FZsE7ZqUyinaSrgcNbGJg7gM1mtiw6PgW4w8x+rUn73AwMpHdo1ktENjonFr2W\nfNcBpzr0m5N3LrAhcbwBODQqNeg5rRyasWTmwoV7k89g/3Pi2cu73gUXXxyey8IuDrJynlMuitSD\naYd67ZidhErqYYJPpjQklzyxql291klyj+Vk1GfRohDlmTGjc9HromuZHCdJLgamlRaMBbmGdnge\nmJI4nhL13fR/dFGCU8lQMQT5gmTmb9IAzJ4dbo89FkLITz4ZRK+feCIYm+9/v33j4CJOTh6UTnAq\n9QDS+2CeNLO/iY5PAW43s4b/44uUa0g6dI88Eq67DhYv3mso6osJ4z2Wp00LRiXexqPd/avjWdH0\n6XDGGS6Z4ORLFTR5h4DJwKeA24ADgKEmbZcAY8AcYCrwH8DfjtN3x+G3tNSn5CfZvt1s1qyQ21Kf\nbxK/1igXZft2s5kzQ9g61vJNM4Y4vyVZDuChaCdPqEAezHJCmHpP4nZ59NqRwC7giET7jwI/Bp6j\nR3kwzWiVbNcs7yWZVDdr1v7i2LGI9tBQawOTbBsbpX4SFHfKTekNTJ63vA1Mq2S7Zhm1rc5rJ4nv\n3//d9knOmznTM3Sd3pHWwFQlTF0qWkk3NJM8aHVeq9fHCz9fd51r2zrlo3Anbx70wsnbafVwq/Oa\nvV4ffl65EpYsgU2bQiRq7Vo3LE7vqEwmbx6UUfS7W0nJZPRpaAjuvz/sueQSCU4RuIEp0efKIvkt\n3vM63uhsZMRnLU5x9FupQKXJQlJyeBg+85lQEAkhOc+lKZ2y4wamB3Si59uIBQuy6cdxeoUvkXpE\nVpKSLk3plIHS+2AkXQT8EfAm4E4bv1TgfOBm4EVCkaMBZ5rZmibtS2dgHKefSGtgiqym3g5cTSQ4\nlaL9Wmu/SNJxnAIpzAdjZvea2TeBnxU1hqJx3Ran36mSk/d4Sc9K2iRpmaTcxt6LH77vhugMAlUR\nnFoNjJjZVklzgXuAl4Frm53QqR5MrwSbXLfFqRKl0oNpR3AqjR5Mg/7PAT5mZm9t8nrHTt5ON39v\nl07Ewx2nLBTq5DWzk/Pot47WYjcdEOesxD/8vHJN4oJIDzk7/UxhSyRJQ8AkgvDUREkHALvNbE+D\ntqcD3zezZyXNBpYBd+cxrl7+8JO7PjpOP1Kkk3cZIa/lUuB90ePLACQdKWmXpCOitu8EfiCpBtwH\nrACuyWtgvg2q42SDZ/I6jtM2XuzoOE7huIFxHCc33MA4jpMbbmAcx8kNNzCO4+SGGxjHcXKjEAMj\n6VWSviRpi6Sdkv47SqYb75ylkp6WtCM6d1Kvxus4TmcUNYOZCGwDTjSz1wCXA/dImtaosaQlwCXA\nycBRwNHAlb0Z6v50UvTl/VfrPbz/bCjEwJjZi2Z2lZn9KDr+NrAZeHOTU94P3Gxmm8xsJ0Go6o97\nM9r9qfqXo+r99+I9vP9sKIUPRtLrgTcAzXTy5wIbEscbgEMlTc17bI7jdE7hBkbSROB24FYze7xJ\ns4OAnYnjnYRqaq8WcpwSU6gejCQBXyUYkPc0qqSO2v0P8EkzWxEd/yrwE+AQM9vRoL0XIjlOzlRB\nD+Zm4BDgjGbGJeJhYB6hihrgOOCZRsYlev9ctGIcx2mPwpZIkm4EZgNnmdkvWjS/DfiApDmR3+Uy\n4Mt5j9FxnO4oKg9mGvAhopmIpFqk/3Ju9Po+ejBmdj/waeC7hGjTZuCKIsbuOE56+lIPxnGcclB4\nFMlxnP5lIAyMpDdI+rmk2zLu9yuSxqJyh02SPpBh322XU3TwHhdJ+p6klyTdkkF/UyV9XdLzkjbH\nS96syHq8DfrvxTXP7TtT9z55fedXRf3uilwbj47XfiAMDPA5YH0O/X4KmB6VO5wFfFLS8Rn13VY5\nRYfE2/fenFF/nwdeAl4H/CFwg6Q5GfUN2Y+3nl5c8zy/M0ny+s4b8BEzm2Jmw2Y27t+37w2MpN8H\ndgD/kXXfZvaomb0cvxXh4h+dUd/tllN08h6Zbd8r6UDgbGCZmf3czB4Cvgmc123fMXlvN9yja57b\ndyYmz+98/BZpG/a1gZE0hVAUeTE57aMk6XpJLwCPAmPAypzep1U5RdG8kbDtzA8Tz20glHlUkryu\neZ7fmV5854From2cH5S0aLyGfW1ggKuAL5rZ9rzewMwuImQivwP4GvB/Wb9HynKKoqkv5yA6rmQ5\nR57XPOfvTN7f+UuAmcDhwBeBb0ma0axxZQ2MpO9KekXSnga3NZLmAb8BXJdH/8m2FlgLHAlcmGX/\nUTnF7YQv4Z/nMf6MeB6YUvfcFKCWw3vlSqfXvB06+c60QtJxdPGdT4OZfc/MXjCzl83sNuAh4Ixm\n7Qvb2bFbWpUjSPpLYDqwLfrCHAQMSTrWzN7Sbf9NmEjK9XQO5RSd9p8VjxN26Dw6sUyaR3mXdOPR\n0TXvkNTfmRQsoovvfIcY4yzFKjuDScFNhD/ccYQv+o2EXSFPy6JzSa+TdI6kV0uaoCCK9ftk6FhT\ne+UUnfQ/JGkyie17Fbb0bRsze5Ew3b9K0oGS3k6IknyljOMd5z1yu+Y9+M7k/Z1/jaTT4usu6X3A\nicD9TU8ys4G4AcuB2zLs7xBgFSGi8RzBoXlBhv1PA14hbKlbi267gHMzviavAHsSt8u76G8q8HXC\ncmkLcE4Of8PMxtvra573d6bJ9cr6O7+e4Fv7GbAWOGW8c7xUwHGc3OjnJZLjOAXjBsZxnNxwA+M4\nTm64gXEcJzfcwDiOkxtuYBzHyQ03MI7j5IYbGMdxcsMNjOM4ueEGxnGc3HAD4zhObriBcXpOVG29\nQtJzki6KnjtP0glFj83JFi92dHqOpCsJFbmbgbcCRwHrzSw3oSSnGNzAOD1H0mvN7Lno8ZuBJWb2\nqcTrMwni4RcUNUYnGyqraOdUl4RxORWYU2dc/oyg4j+9oOE5GeI+GKcQFDZle72Z/VPyeTP7HHBr\nIYNyMscNjNNzJJ0PPG9mtyeeu6bAITk54Uskp6dI+i3gPcAWSYcTJCnPI+jHOn2GGxinZ0QbmR1j\nZmdHe+ncS9B5/biZfaPY0Tl54AbG6Rlm9gzwd9HjzQTle6ePcR+MU0ZEftueOj3EDYxTKiR9EPgY\n8CZJn5T0hqLH5HSOJ9o5jpMbPoNxHCc33MA4jpMbbmAcx8kNNzCO4+SGGxjHcXLDDYzjOLnhBsZx\nnNxwA+M4Tm78P0hbAbrwbnShAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f887170c390>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure(figsize=(4,3))\n",
    "plt.plot(codings_val[:,0], codings_val[:, 1], \"b.\")\n",
    "plt.xlabel(\"$z_1$\", fontsize=18)\n",
    "plt.ylabel(\"$z_2$\", fontsize=18, rotation=0)\n",
    "save_fig(\"linear_autoencoder_pca_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Stacked Autoencoders"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Let's use MNIST:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/data/train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"/tmp/data/\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Train all layers at once"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Let's build a stacked Autoencoder with 3 hidden layers and 1 output layer (ie. 2 stacked Autoencoders). We will use ELU activation, He initialization and L2 regularization."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Note: since the `tf.layers.dense()` function is incompatible with `tf.contrib.layers.arg_scope()` (which is used in the book), we now use python's `functools.partial()` function instead. It makes it easy to create a `my_dense_layer()` function that just calls `tf.layers.dense()` with the desired parameters automatically set (unless they are overridden when calling `my_dense_layer()`)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "from functools import partial\n",
    "\n",
    "n_inputs = 28*28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 150  # codings\n",
    "n_hidden3 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01\n",
    "l2_reg = 0.0001\n",
    "\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer() # He initialization\n",
    "#Equivalent to:\n",
    "#initializer = lambda shape, dtype=tf.float32: tf.truncated_normal(shape, 0., stddev=np.sqrt(2/shape[0]))\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "\n",
    "my_dense_layer = partial(\n",
    "    tf.layers.dense,\n",
    "    activation=tf.nn.elu,\n",
    "    kernel_initializer=initializer,\n",
    "    kernel_regularizer=tf.contrib.layers.l2_regularizer(l2_reg))\n",
    "\n",
    "hidden1 = my_dense_layer(X, n_hidden1)\n",
    "hidden2 = my_dense_layer(hidden1, n_hidden2)\n",
    "hidden3 = my_dense_layer(hidden2, n_hidden3)\n",
    "outputs = my_dense_layer(hidden3, n_outputs, activation=None)\n",
    "\n",
    "mse = tf.reduce_mean(tf.square(outputs - X))\n",
    "\n",
    "reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)\n",
    "loss = tf.add_n([mse] + reg_losses)\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now let's train it! Note that we don't feed target values (`y_batch` is not used). This is unsupervised training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train MSE: 0.025843\n",
      "1 Train MSE: 0.0132539\n",
      "2 Train MSE: 0.0115193\n",
      "3 Train MSE: 0.0110359\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 4\n",
    "batch_size = 150\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = mnist.train.num_examples // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch})\n",
    "        mse_train = mse.eval(feed_dict={X: X_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_train)\n",
    "        saver.save(sess, \"./my_model_all_layers.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "This function loads the model, evaluates it on the test set (it measures the reconstruction error), then it displays the original image and its reconstruction:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def show_reconstructed_digits(X, outputs, model_path = None, n_test_digits = 2):\n",
    "    with tf.Session() as sess:\n",
    "        if model_path:\n",
    "            saver.restore(sess, model_path)\n",
    "        X_test = mnist.test.images[:n_test_digits]\n",
    "        outputs_val = outputs.eval(feed_dict={X: X_test})\n",
    "\n",
    "    fig = plt.figure(figsize=(8, 3 * n_test_digits))\n",
    "    for digit_index in range(n_test_digits):\n",
    "        plt.subplot(n_test_digits, 2, digit_index * 2 + 1)\n",
    "        plot_image(X_test[digit_index])\n",
    "        plt.subplot(n_test_digits, 2, digit_index * 2 + 2)\n",
    "        plot_image(outputs_val[digit_index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure reconstruction_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAd8AAAGoCAYAAAAHJ+8hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHWZJREFUeJzt3VuMnVX5P/DVczszPU0PtAUKLRTaYpCjVE5BIRhjIlHk\nAjHRoInERBNJ1EBiot7onXJj0GhAwQsTE1FDOElCAgYEkYMBArSAQtuhLVDa6XSmJ343/4u/yXqW\nvJvN02n7+Vw+b9fae/aezjdv8jzvmvLee+8VACDP1MP9BgDgWCN8ASCZ8AWAZMIXAJIJXwBIJnwB\nIJnwBYBkwhcAkglfAEg2PfG1PEqLo8mUw/0GjlZbt26t/q1oPY1vypTuX0e0X2uvrmt6ec+9PHWw\nnz9/L3r5zI4VK1asqH447nwBIJnwBYBkwhcAkglfAEiW2XAF8D/1s0Enq+Gpn2v63bx0rDc8RfrZ\nWNcLd74AkEz4AkAy4QsAyYQvACQTvgCQTPgCQDKjRsAR70gcwcl6/YMHD1br06ZNq9b37dsX7jVz\n5sxqferU+D4uGs85dOhQuKbrZzOZR8oi7nwBIJnwBYBkwhcAkglfAEgmfAEgmW5n4KjV70MKur5O\nL124vbznAwcOdF6zZ8+eaj3qji4l7lAeGBjo/PqtDuloTS/fWT+7nfvJnS8AJBO+AJBM+AJAMuEL\nAMmELwAkE74AkMyoEXDU6vcD97uOp8yYMSO8Fh1s0BK959ao0fj4eKfXaI0a7d27t1rfv39/uGbe\nvHnVemukKvrcolGn1iEN0c+TNeoUcecLAMmELwAkE74AkEz4AkAy4QsAyXQ7A0eEVndspJfO5Va3\nb9ShHO3V6gKOXmfmzJnhmunT63+yW53TExMT1frg4GC1HnUnl1LKvn37qvVWt3H0vbV+zq6d4FEX\ndut1op+llP4elBFx5wsAyYQvACQTvgCQTPgCQDLhCwDJhC8AJDNqBBwRehnzaI1/RIcRtMaDxsbG\nqvVo1KU1zhKNGg0NDYVrFi5cWK3PmjUrXLNgwYJqfc6cOdV6L2NLrZ8z+t5a303XAwyiEayWXsbQ\n+vme3fkCQDLhCwDJhC8AJBO+AJBM+AJAMt3OpZTHHnusWr/llluq9eOPPz7cK+og/PKXvxyuGR4e\n7lQH/lvXTtNS4q7mXbt2hWtef/31an1kZKRaf/nll8O9ove8aNGicM38+fOr9agLurUmOkChdbBE\n1FXcOlghujY6OhquiX6exYsXV+vRIRGt1+/ld2bq1Ph+1cEKADDJCV8ASCZ8ASCZ8AWAZMIXAJJN\n6aXjq0dpL9TV6aefXq23OhX7KepG3LBhQ8rrZzn55JOr9ZtuuqlaX7ly5Yf4bj6wbq2NvG9btmzp\n/Lci6tCNnt/curZjx45wzUsvvVSt//3vf6/WX3zxxXCv6HnMrSmHqKP2jTfeCNdEHcqzZ8+u1rdv\n3x7uFX1m0d/QUkpZvnx5tf7mm292XnPJJZdU62vWrAn3iiZQWqKu5tZzryMrVqyofmnufAEgmfAF\ngGTCFwCSCV8ASCZ8ASCZ8AWAZA5WKKXcdddd1frTTz9drZ9xxhnhXs8991y1Ho0ilFLKn/70p2r9\nvvvuC9esWrWqWn/11VfDNV1FIwqlxKMA0YPnW6IRpO9973ud9+LIF43TtMYioxGQ1qjR2NhY59eJ\n3tvAwEC1fsopp4R7LV26NLwW2bRpU7XeGqeJrkWfTWvUKDqMIfr5S4nHoB555JFwzVlnnVWtr169\nulpfu3ZtuFf0nc2YMSNc08thDA5WAIBJTvgCQDLhCwDJhC8AJBO+AJBMt3MpZd26dZ3qLWeeeWa1\nfu2114ZrfvKTn1Trr732Wrgm6nZ+5ZVX4jfX0cyZM8NrUbdz9L5KibsoW52KHHt6OewlWhM9IL+U\n+ECT6MCDUko59dRTq/XFixdX662O2qg7ds+ePeGa6LCR1gP/Fy5cWK2Pj49X61dccUW4V/SZRfVS\n4mmO0dHRcM3OnTur9eh7bn3O0d+xfnYu97LGnS8AJBO+AJBM+AJAMuELAMmELwAkE74AkMyo0SQw\ne/bsar2XEZxexqN6ER0UsWPHjnDNBRdcUK1feeWVfXlPHB16OVghWtMaNYoMDg6G16IDBKLDQVqv\n38uo0emnn16tt0Z9ojGc3bt3V+u9jGe9/fbb4Zro4JrWeFT0HqKDFVpjkdFe0eEJpfQ27taVO18A\nSCZ8ASCZ8AWAZMIXAJIJXwBIptuZUKvr8nOf+1y13uog/NnPflatz5kzp9sb46jWz07TXjpqp0+P\n/yxG+0X1VufwwYMHO9VLif+vzJ07N1yzf//+TnsNDQ2Fe0WTGf/617/CNSMjI9X6okWLwjWf+tSn\nqvVly5ZV6710O7dEh070kztfAEgmfAEgmfAFgGTCFwCSCV8ASCZ8ASCZUSNCt99+e3itl/GBk046\n6YO+Jeikl4MNehlPikbsxsbGwr2iNa3Xj9a0RmOikcHodVrjgps3b67W77zzznDN888/X61fcskl\n4ZpLL720Wl+xYkW13stIV+vn7EXXETl3vgCQTPgCQDLhCwDJhC8AJBO+AJBMtzNl06ZN1fqNN97Y\nea9HH300vBY9FB3ej6g7ud/27dsXXou6aqPDC3bv3h3uFR1S0MvBDu+88064Juq4XrBgQbU+Ojoa\n7nX//fd3qpdSyqxZs6r18847L1yzcuXKan3GjBnVeuswiqirudXtHP2utX4HdTsDwCQnfAEgmfAF\ngGTCFwCSCV8ASKbbmfKXv/ylWo86OEsp5ZprrqnWV69e3Zf3xLEr6ihtdZP2c02r2zj6PxE9P/nA\ngQPhXtF762VNqws3et76vHnzqvUdO3aEez3++OPV+t69e8M1Z555ZrV+zjnnhGvmzp1brffymUVa\na6Ku8q4dzS3ufAEgmfAFgGTCFwCSCV8ASCZ8ASCZ8AWAZEaNjhGtsaE//vGP1Xr0QPRSSvnxj39c\nrUct+vB+9TLO0csITi9jK9HD+KMDF2bOnBnuFV2L9iol/n8cHThQSinDw8OdXufVV18N99q8eXO1\nvmTJknDNpZdeWq2vWbMmXBO9t+iQiJbW59lPXQ/+cOcLAMmELwAkE74AkEz4AkAy4QsAyXQ7HyN+\n/etfh9cefvjhav2LX/xiuMYBChwJou7k1rVWx3506MKcOXM6/ftSSjl48GB4LTI+Pl6tz549O1wT\n/Zxbtmyp1v/85z+He23cuLFav+yyy8I1n/70p6v1wcHBcM3o6Gi1Hh3g0Pqco07w1vccdcK3Oqe7\ndum78wWAZMIXAJIJXwBIJnwBIJnwBYBkwhcAkhk1Oso8/fTT1fo3v/nNcM2CBQuq9R/96Ed9eU8w\nGUWjIfv27QvXRGMrAwMD1fqePXvCvaJRo9brR4cxDA0NhWvefvvtav3ee++t1u+6665wr+jnPP/8\n88M1S5curdZbY2DRZxONFLUOlogOymgdhBCNIXU9PKHFnS8AJBO+AJBM+AJAMuELAMmELwAk0+18\nhIoeMH7ttddW662HuF933XXVusMTOFJEncu9HKzQekB+tGZiYqJve7Ue3j9r1qxOe5VSyiOPPFKt\nR4etjIyMhHt96UtfqtbPOuuscE2k9dlEXcXRzx91gZdSyv79+7u9sRK/t66HJ7S48wWAZMIXAJIJ\nXwBIJnwBIJnwBYBkwhcAkhk1msRa4wOf+cxnqvUXX3yxWl+3bl241w9/+MNubww+RP18eH1L9P8r\nGmcpJR4pih7s3xpzicb/WockRNc2btwYrrn//vur9Zdeeqlav/zyy8O9rrrqqmp9/vz54ZpoLDI6\nvKCUeNyql8Mwot+n1t/XaI1RIwA4gglfAEgmfAEgmfAFgGTCFwCS6XaexN5+++3w2kMPPdRprzvu\nuCO8Njw83Gkv+DD10lEadae2DimIjI+Ph9emT6//yYw6elud27Nnz67WBwcHwzVjY2PV+qZNm8I1\n0cEKxx9/fLX+hS98IdzrvPPOq9Zbn/Pu3bur9Tlz5oRros856hBvdS5He7V+z6Kfp/V9dv29decL\nAMmELwAkE74AkEz4AkAy4QsAyXQ7TwLvvvtutb5hw4bOe915553V+tlnn915LzjStbpTBwYGqvXo\n+c2lxM99jrptW1240fOgW1MOr776arUe/b8vpZTNmzdX69ddd121ftlll4V7RZMR77zzTrgm6hxu\nPfc6+txmzpxZrbeeEx3pZ+fy/9qvxp0vACQTvgCQTPgCQDLhCwDJhC8AJBO+AJDMqNEkcNttt1Xr\nr7zySue9Lr744mq9axs8HKuicaJS4of0R2NDrf930ahNa2znnnvuqdYfeOCBcE00hrNq1apqvXXg\nwYEDB6r10dHRcE0kOliilO4HK7S+s2hNVC8l/t56GUGKuPMFgGTCFwCSCV8ASCZ8ASCZ8AWAZLqd\nk7z88svhtR/84Ad5bwQmuV46TXvpQm0dehCJun2jwwNaD/yPOoQff/zxcM1jjz1Wre/Zsydcs2bN\nmmp99erV1XqrQ3t8fLxajw48aGm9TvR5RmtahzT0op9dzRF3vgCQTPgCQDLhCwDJhC8AJBO+AJBM\n+AJAMqNGSR5++OHw2q5duzrvt27dumq99VB0OBJEYx6t0ZSM0ZCWaASpNc60e/fuar11SMHJJ59c\nrc+dOzdcc8YZZ1TrK1eurNajMZ9S4s85OliitaaXgw16+ffR6/fyO9PPA2rc+QJAMuELAMmELwAk\nE74AkEz4AkAy3c6T2IUXXhhee+CBB6p13c4crfrdndrPDumoq7nV7Ry9t6g7uZRSjjvuuGo96pwu\npZTTTjutWl+yZEm13s8u5FLiz6DVVd3L4RqRrE74rq/jzhcAkglfAEgmfAEgmfAFgGTCFwCSCV8A\nSDYl8YHkh/fJ59Bf/XvCOv9l69at1b8Vh/vwhJZ+juC0fs5oDKg10jR9en2idNq0adV6v8ezevls\nuh6u0XpfGa/fsnz58uoid74AkEz4AkAy4QsAyYQvACQTvgCQLLPbGQAo7nwBIJ3wBYBkwhcAkglf\nAEgmfAEgmfAFgGTCFwCSCV8ASCZ8ASCZ8AWAZMIXAJIJXwBIJnwBIJnwBYBkwhcAkglfAEgmfAEg\nmfAFgGTCFwCSCV8ASCZ8ASCZ8AWAZMIXAJIJXwBIJnwBIJnwBYBkwhcAkglfAEgmfAEgmfAFgGTC\nFwCSTU98rfcSXws+bFMO9xs4Wo2MjBzWvxXvvRe//JQpvvbD6dChQ9X61KmT9z5y2bJl1V+ayfuO\nAeAoJXwBIJnwBYBkwhcAkmU2XAFMeoe7qarVPHTw4MFq/XC/55ZWA1tX0c95JDbJufMFgGTCFwCS\nCV8ASCZ8ASCZ8AWAZMIXAJIZNQI4DKLnFB84cCBcE40atURjODNnzqzWp02b1nmv1vuK1rRGgPr5\nrOaMUadeuPMFgGTCFwCSCV8ASCZ8ASCZ8AWAZLqdk/zud78Lr+3Zs6daf/LJJ8M1v/zlLzu9/ve/\n//3w2ic/+clq/bLLLuv0GnA0iLqQS4k7Z6M1+/fvD/eK/t/v3bs3XLNr165qfefOneGa6D3MmTOn\nU72UUgYGBqr1uXPnhmt66aqO3sP06fXI6md3dCn97ZCOuPMFgGTCFwCSCV8ASCZ8ASCZ8AWAZMIX\nAJJNyWip/n/SXuhw+sY3vlGt/+IXv0h+J+/f+vXrq/VHHnkkXDN//vwP6+0cKfr3hHX+y8jISPVv\nRetvVT8feN86JGDfvn3V+rvvvlutv/POO+FeGzdurNafeOKJcM2zzz7baa9S4vGgoaGhar01NrRu\n3bpq/aKLLgrXbNiwoVofHh4O18yYMaNaj76b1vcfXWuNJ0Wv08vv4LJly6oX3PkCQDLhCwDJhC8A\nJBO+AJBM+AJAMgcr9CDqaC6lv13NZ599dnjt6quvrtZffvnlav03v/lNuNfzzz9frf/hD38I13z1\nq18Nr8GHodXR2s+pjQMHDoTXRkdHq/UdO3ZU6//4xz/CvZ566qlqvdXtHB3g0PpbsWrVqmp9fHy8\nWn/66afDvaKu6jVr1oRrooMVFixYEK6JvoOJiYlO/76l1dXei64d9+58ASCZ8AWAZMIXAJIJXwBI\nJnwBIJnwBYBkRo0a/vOf/1Trv/rVrzrvdf7554fX7r333mo9eiB6KXH7ftQ+33rw+t/+9rdqPRqf\ngKNZLwcrjI2NdaqXEo9HXXDBBeGaK664olpvHWwwODhYrUcHp0Q/YymlbNu2rVqPDkIoJf459+7d\nG66JRKNG0dhUr2bNmlWtT5/ev8h05wsAyYQvACQTvgCQTPgCQDLhCwDJdDs3RN2+rYe4R13Nf/3r\nX8M1Q0ND3d5Yw+23316ttx7WHrnqqqs+4LuB7qL/X60H10fXWmuiztX9+/eHa6Ipg4ULF1brq1ev\nDvdatmxZtf6Rj3wkXLN+/fpqffHixeGa7du3V+vRNEfrb8Xw8HC1Hv38pcSdw62u6uighOh3o/Wd\nRXtF76uU3n6funLnCwDJhC8AJBO+AJBM+AJAMuELAMl0Ozecc8451XrrmcdRN+ScOXP68p7+l+i5\n063OQphM+tpROjW+v4ieR9yaPujabXvyySeHe82ePbtaP+GEE8I18+bNq9Z3794drnn22Wer9ejZ\nzlF3dClxt/Upp5wSrullmiN6vnb0nR06dCjcK+p2njZtWrimNdHSL+58ASCZ8AWAZMIXAJIJXwBI\nJnwBIJnwBYBkRo16MH/+/MP9Fsodd9xRrT/zzDOd97ryyiur9db4ABzpxsbGqvXBwcFwzdy5c6v1\n6JCG6N+XEo8lttZEYzM7d+4M1zz55JPVejRS1DqkYe3atdV6azwqOsBg79694Zro84wMDAyE16LP\nLBpnKiUeUevnCJI7XwBIJnwBIJnwBYBkwhcAkglfAEim23kSe+qpp8JrX//616v1iYmJan358uXh\nXrfccku1Hj3EHI4Ure7U6GCD1pqoCzbaq/Xw/uj/auuQgNHR0Wr93//+d7jmiSeeqNZfe+21an3D\nhg3hXhdccEG13vpbEXViR93mpcQd51HndEt0qE3rsJnWd9Av7nwBIJnwBYBkwhcAkglfAEgmfAEg\nmfAFgGRGjSaxRx99NLwWjSlEbrjhhvDaaaed1mkvOFJMmTIlvBaNFLXGTKJrrZGirq/fGoHZunVr\ntX7fffeFazZt2lStL1u2rFpft25duNdJJ51UrbdGgHbv3l2tRwdLlBJ/BtFIU2s8LFrTy+9GNGr2\nv/ar7tXpXwMAH5jwBYBkwhcAkglfAEgmfAEgmW7nSeD666+v1n//+9933uvb3/52tf7d7363815w\nNDtw4EC13ktH6/79+/u21/bt28M19957b7X+wgsvhGvmzp1brZ9zzjnV+rnnnhvuFXVIHzx4MFwT\ndRsPDAyEayLR59zLITDRgQulxB3a/eTOFwCSCV8ASCZ8ASCZ8AWAZMIXAJIJXwBIZtQoyejoaHjt\nnnvuqdbHx8fDNccdd1y1fvPNN1frrYeYw9Gq9cD9rg/CL6V96EJN6/9dNDbz3HPPhWsefPDBan3n\nzp3hmo997GPV+sUXX1ytRyNIpZSyaNGian3z5s3hmtmzZ1frrfGgsbGxaj0aD2sdbNEag4r08rvR\nlTtfAEgmfAEgmfAFgGTCFwCSCV8ASKbbOck111wTXtu2bVvn/b71rW9V68PDw533gqNVq9O1dehB\nV1FXdas7Onp4/1NPPRWuiTqhowMPSill9erV1fratWur9YULF4Z7vfXWW9V61J1cSikLFiyo1lud\n6FFXc9Qh3fouo9fZu3dvuCb63lpd1V258wWAZMIXAJIJXwBIJnwBIJnwBYBkup377Mknn6zWH3ro\noc57ff7znw+v3XjjjZ33gyNZL89pbj2jN+qEbj1zONL1WcSllPLCCy9U6w8//HC4JnqG84UXXhiu\n+ehHP1qtDw4OdnqNUkrZsmVLtT40NBSumZiYqNajZ1uXEn/X0bOyp0+Poyx6Rn70vkrp7XegK3e+\nAJBM+AJAMuELAMmELwAkE74AkEz4AkAyo0Y9aD2Q+6abbqrW9+3b1/l1zj333PBa1HIPR6vW2FA/\ntcZMogfuR2Mz0ThRKaXcfffd1Xp0eEIppSxdurRa/8QnPhGuWb9+fbU+MDBQrY+OjoZ7zZ8/v1pv\nHcYQjQG1Ro2iMahor9ZIVzRq1Dp0Y9asWdV6P38H3fkCQDLhCwDJhC8AJBO+AJBM+AJAMt3OPbj1\n1lvDaw8++GDn/a6//vpq3eEJ8P5EXaitwxhmz55drbce0h911UYHK7z44ovhXv/85z+r9ajTt5RS\nrr766mr98ssvD9dEHcpR53ar23v58uXVemv6Ys+ePZ1fJ+o2jvZ68803w72irurW5xz9brS6nbt2\nQrvzBYBkwhcAkglfAEgmfAEgmfAFgGTCFwCSGTXqwc0339zX/X76059W6w5PgA9m2rRp4bVopKiX\nUaNdu3ZV61u2bAn3evfdd6v1E088MVxz4YUXVutLliwJ10RjUNGBAxMTE+Fe0WEMU6fG93HRAQbR\n65cSH16zdevWaj36GUuJR6qicaIs7nwBIJnwBYBkwhcAkglfAEgmfAEgmW7nSWB0dLRab3UQ9lP0\nEPNWp2jUwdjqlIxEnY233HJL571aop+n1b3eevg7k0d0gELr/1D0+xB1x5YSP6T/jTfeqNZbD/yP\nOqTnzZsXrtm4cWO1fvzxx4drosMItm/f3unfl1LK4sWLq/W5c+eGa3bv3l2ttw4ieOWVV6r1559/\nvlpftGhRuNfatWur9RUrVoRroq721t9EBysAwCQnfAEgmfAFgGTCFwCSCV8ASCZ8ASCZUaNJoDUm\nkOGGG26o1lut+CMjI9X6z3/+8768p0ytz/9rX/ta4juh31pjQ9G4XOtghegwgGhcb9u2beFe0RhS\nNM5USil33313tf7EE0+Ea6LRoR07dlTrq1atCveK3lvrkIJoPCcasSwlHoOKxhLPP//8cK9INJ5W\nSvexoV648wWAZMIXAJIJXwBIJnwBIJnwBYBkup17cN1114XXbrvttsR30h+33npryutEXaSth5VH\nvvKVr1TrH//4xzvvddFFF3Vew5Eh6mguJe6EjjpqS4m7YIeHh6v19evXh3u99dZb1frrr78ertm8\neXO1Hk0ftK5Fncut148OHDjllFPCNdFnEx0sUUr83axcubJab00sRIcutLrao79J/eyCducLAMmE\nLwAkE74AkEz4AkAy4QsAyaa0nm/ZZ2kvdDj99re/rdb37dvX19d55plnqvV+Plv5O9/5Tnjt1FNP\n7bzfZz/72Wp96dKlnfeaBD78h78eo0ZGRlL+VkTdrlOndr8niTqXX3vttXBN9H+49czjqBO71Tkc\nPSc5EnU0l1LKwoULq/Vly5aFa2bOnFmtR8/JLqWUOXPmVOsnnnhitX7CCSeEe0V/X+bNmxeuiZ5V\n3ctkxrJly6p/K9z5AkAy4QsAyYQvACQTvgCQTPgCQDLhCwDJjBpBb4wafUiyRo2ikaLWA/ejNdEh\nBa0Rw+gh/QMDA+Ga6HVaYzvbtm2r1sfGxjrvFY3ntMazorGhVvZE16LvZtasWeFevXzPM2bMCK91\nZdQIACYJ4QsAyYQvACQTvgCQTPgCQLK43QvgKHbo0KFqPeoobok6altduJFeuq2HhobCNUuWLOn0\nOq1u56gLOfosS4kPI2gd4BC9TvTdHDx4MNxrsnLnCwDJhC8AJBO+AJBM+AJAMuELAMmELwAkM2oE\n8P/p5bCZaNSltVc0ahMdeFBKPB7UGk+KrkUjQC3RSFF0SEQp8WfTGk/qZaQp0npvXV+/l70i7nwB\nIJnwBYBkwhcAkglfAEgmfAEgmW5ngA+ol+7YGTNmVOutQwKibt/Wmug9TExMhGu67hUd+NCrXjrO\n+6mfXc0Rd74AkEz4AkAy4QsAyYQvACQTvgCQTPgCQDKjRgAfUF8fuN/nsZ1oDKmf77mXAw/6KWM0\nqN/c+QJAMuELAMmELwAkE74AkEz4AkCyKYf7AdYAcKxx5wsAyYQvACQTvgCQTPgCQDLhCwDJhC8A\nJBO+AJBM+AJAMuELAMmELwAkE74AkEz4AkAy4QsAyYQvACQTvgCQTPgCQDLhCwDJhC8AJBO+AJBM\n+AJAMuELAMmELwAkE74AkOz/AP81b0yongN2AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f885c05db00>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs, \"./my_model_all_layers.ckpt\")\n",
    "save_fig(\"reconstruction_plot\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Training one Autoencoder at a time in multiple graphs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "There are many ways to train one Autoencoder at a time. The first approach it to train each Autoencoder using a different graph, then we create the Stacked Autoencoder by simply initializing it with the weights and biases copied from these Autoencoders."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Let's create a function that will train one autoencoder and return the transformed training set (ie. the output of the hidden layer) and the model parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from functools import partial\n",
    "\n",
    "def train_autoencoder(X_train, n_neurons, n_epochs, batch_size, learning_rate = 0.01, l2_reg = 0.0005, activation=tf.nn.elu):\n",
    "    graph = tf.Graph()\n",
    "    with graph.as_default():\n",
    "        n_inputs = X_train.shape[1]\n",
    "\n",
    "        X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "        \n",
    "        my_dense_layer = partial(\n",
    "            tf.layers.dense,\n",
    "            activation=activation,\n",
    "            kernel_initializer=tf.contrib.layers.variance_scaling_initializer(),\n",
    "            kernel_regularizer=tf.contrib.layers.l2_regularizer(l2_reg))\n",
    "\n",
    "        hidden = my_dense_layer(X, n_neurons, name=\"hidden\")\n",
    "        outputs = my_dense_layer(hidden, n_inputs, activation=None, name=\"outputs\")\n",
    "\n",
    "        mse = tf.reduce_mean(tf.square(outputs - X))\n",
    "\n",
    "        reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)\n",
    "        loss = tf.add_n([mse] + reg_losses)\n",
    "\n",
    "        optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "        training_op = optimizer.minimize(loss)\n",
    "\n",
    "        init = tf.global_variables_initializer()\n",
    "\n",
    "    with tf.Session(graph=graph) as sess:\n",
    "        init.run()\n",
    "        for epoch in range(n_epochs):\n",
    "            n_batches = len(X_train) // batch_size\n",
    "            for iteration in range(n_batches):\n",
    "                print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "                sys.stdout.flush()\n",
    "                indices = rnd.permutation(len(X_train))[:batch_size]\n",
    "                X_batch = X_train[indices]\n",
    "                sess.run(training_op, feed_dict={X: X_batch})\n",
    "            mse_train = mse.eval(feed_dict={X: X_batch})\n",
    "            print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_train)\n",
    "        params = dict([(var.name, var.eval()) for var in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)])\n",
    "        hidden_val = hidden.eval(feed_dict={X: X_train})\n",
    "        return hidden_val, params[\"hidden/kernel:0\"], params[\"hidden/bias:0\"], params[\"outputs/kernel:0\"], params[\"outputs/bias:0\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now let's train two Autoencoders. The first one is trained on the training data, and the second is trained on the previous Autoencoder's hidden layer output:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train MSE: 0.0193756\n",
      "1 Train MSE: 0.019141\n",
      "2 Train MSE: 0.018939\n",
      "3 Train MSE: 0.0192285\n",
      "0 Train MSE: 0.00419925\n",
      "1 Train MSE: 0.00431123\n",
      "2 Train MSE: 0.00460423\n",
      "3 Train MSE: 0.00459113\n"
     ]
    }
   ],
   "source": [
    "hidden_output, W1, b1, W4, b4 = train_autoencoder(mnist.train.images, n_neurons=300, n_epochs=4, batch_size=150)\n",
    "_, W2, b2, W3, b3 = train_autoencoder(hidden_output, n_neurons=150, n_epochs=4, batch_size=150)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Finally, we can create a Stacked Autoencoder by simply reusing the weights and biases from the Autoencoders we just trained:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "n_inputs = 28*28\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "hidden1 = tf.nn.elu(tf.matmul(X, W1) + b1)\n",
    "hidden2 = tf.nn.elu(tf.matmul(hidden1, W2) + b2)\n",
    "hidden3 = tf.nn.elu(tf.matmul(hidden2, W3) + b3)\n",
    "outputs = tf.matmul(hidden3, W4) + b4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa0AAAFxCAYAAADAqvdjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGqxJREFUeJzt3UusnWX1P/Cn0Pv9tLS09pSCFIViNKBGRGOUAROjRg0D\nZKBBE4mJJpKogcREnehMmRg0GlB0YGKixoEakSDIxQvhkkCAQltooeW00NLL6YUCv/n/Weuf/bbn\nnHa1n89wZe2937N3z1l5s79dz6y33367AUAF55zqCwCAURlaAJRhaAFQhqEFQBmGFgBlGFoAlGFo\nAVCGoQVAGYYWAGXMnsHXsnqDmTDrVF/A2WDv3r1+n5lWY2Nj4e+yOy0AyjC0ACjD0AKgjJn8Tgvg\ntDFrVvz1Z3TyxZDe6RK91jnnjH7fcaac6OFOC4AyDC0AyjC0ACjD0AKgDEMLgDKkB4EzXpScG5Km\nG9IbJfqylF/0vLNnx3+Wo9633nprWnqztGRkSO9UcKcFQBmGFgBlGFoAlGFoAVCGIAZwWstCEFEA\nIOt98803T+oazj333LA+akAjC2JEzzsktJGJQhfZ877xxhtdbabDFUO40wKgDEMLgDIMLQDKMLQA\nKMPQAqAM6UHgtBGl1rIkW5SQy1YVRbJEYZSyG3INx48f72rz5s07qcdn15WlGqNVUEMSgVGicKjo\neqfiIEp3WgCUYWgBUIahBUAZhhYAZQhiANNqyLqlKJgwJASRhSuycMOo9u/fH9ajIMSiRYu6WhYQ\niQIPc+fODXujYMOcOXNGvq4sXBG9Z8eOHQt7s7O+ItnaqJPlTguAMgwtAMowtAAow9ACoAxDC4Ay\npAeBaRUlBbP0YJQ4m5ycDHuHpAcPHz480mu1NmwNU/R60c+WpQeH/AxRqnDp0qVhb3S9WXoweh8W\nL14c9kbvTXa90fuQJUGHrHdypwVAGYYWAGUYWgCUYWgBUMZZFcR4+OGHu9ptt90W9q5bt66rLViw\nIOz94he/2NVWrFgR9mZ1OFMN+ZI9CkwcPHgw7I3q2bqmqDcLeEQrjBYuXBj2RvV9+/Z1tW3btoWP\nj0IQu3fvDnuj0MaqVavC3uhv1aZNm8LeCy64oKstX7487I3WQ2VBjKiehV+GnPXlTguAMgwtAMow\ntAAow9ACoAxDC4AyZg1J9pykGXuhzLvf/e6utnnz5ml5rWXLloX1q666alpeb7pceOGFXe2WW24J\ne6MU0ikwegyJE7Z3796Rf5+jNN6hQ4fC3ijll/Xu3bu3qz377LNh7yuvvDLSdbUWp/Syv5NRUnDX\nrl1d7YUXXggfHx2qmP3tOHr0aFiPrFy5squ9973vDXuvueaarnbZZZeFvVFaMkoUtjZsjVNUHxsb\nC5vdaQFQhqEFQBmGFgBlGFoAlHFWrXH64x//2NUee+yxsPfyyy/vak8++WTY++9//7ur/elPfwp7\n//a3v3W1iy66KOzdunVrWB9V9CVva62tXbu2q23fvn3k543CGa219p3vfGfk5+DsEa1Wys52OnLk\nSFfLAgjRuqNs5dPExMTIzxv93gw5DytaVZStRTrvvPNG7t2zZ09Xe+aZZ8Lexx9/vKstWrQo7P3Q\nhz7U1aKQS2vxezN//vywNwtoRJynBcAZydACoAxDC4AyDC0AyjC0ACjjrEoPRqtJsnUlkWwNyvXX\nX9/VfvSjH4W90WFwWXpwy5YtI19bZO7cuWE9Sg9m1xAltC699NKTui7OTFkCLKpn/zaXLl3a1bLk\n3uLFi7tatL4oe44sPRil4cbGxsLeKBl54MCBrpatL9q4cWNXyw6b/ec//9nVskRzlM4csg4rO6wx\net45c+aEvdHPPOSwx4w7LQDKMLQAKMPQAqAMQwuAMs6qIMZMylabDAkxDAmJDBGtnYpWxLQWr3i5\n9tprp/yaqC/78j4KXWS/H/PmzRv5eaOVQqtXrw57o8BEdp5WdGZUdsZVFEyIwgbZ2qpojVPW+9BD\nD3W17GdYv379SLXW4vcsC4NEn0X03ma9Q1Y7ZdxpAVCGoQVAGYYWAGUYWgCUYWgBUIb04BksW9vy\n2c9+tqtlq3J+8pOfdLUsWcTZLVvjFKX8snU+UeIsO7zwzTff7GrRaqfsebN/80uWLBm5N/qZo97s\nZ4iSlTt37gx7X3311bAeiVLKV1xxRdi7atWqrpZ9Ptn6rchUrGyKuNMCoAxDC4AyDC0AyjC0AChD\nEOMMduedd4b1Xbt2dbXsHKINGzZM5SVBay0PbURf9GeBomi1UrYeKlofFAU5WovPh8rO3orCBtFr\nZaGEaAXS1q1bw97o3KvsHLwoiHHxxReHvdFZYdn5YVGgJXsfs/VbJ8udFgBlGFoAlGFoAVCGoQVA\nGYYWAGVID54hnn/++a528803j/z46IC51lpbs2bNCV8TtBYn57L0YFTPEoGRbMXYqIc1Zr0HDhwI\ne6MVVUPSg9Hhq//5z3/C3omJia62fPnysDdKFWaJwOjQy+gwztbiFVVDPsupWO3kTguAMgwtAMow\ntAAow9ACoAxBjDPEn//8564WfaHcWmvXXXddV3vnO9855dcEmewL+aie9UYhiCGiFUqttTY5OdnV\nonBFdg1RiCF6ztZae+mll0aqtdba7t27u9r69evD3vHx8a6WhTZWrFjR1bKfNwpiZGucsuc4We60\nACjD0AKgDEMLgDIMLQDKEMQoJgtX/OEPf+hq2f9q/+EPf9jVputLUxgi264QGXK205EjR7ra4cOH\nw97odyw7Gyra1hE97/bt28PH33333V3tiSeeCHujn23t2rVh7/nnn9/VosBFa3GYZMjnkBlyntag\nz/1ELgYATgVDC4AyDC0AyjC0ACjD0AKgDOnBYn75y1+G9fvvv7+rfeELXwh7rWziVMtWMw1JBEaO\nHTsW1g8ePDhy75IlS7ra4sWLw97oeqNVR48++mj4+Ogcu+jcrNZau/jii7vaVVddNXLvkHPJsvc8\n+tmiWibrHXLOljstAMowtAAow9ACoAxDC4AyBDFOY4899lhX+/rXvx72Rmfl/OAHP5jya4KhhnzJ\nHn1Rn60DitYlDVkHNGfOnLAerTXKfoYosPDss892tYcffjh8/AsvvNDVorOwWmvtU5/6VFe78sor\nw965c+d2tey9id7zaO1Va/kauVGfd8i/hYw7LQDKMLQAKMPQAqAMQwuAMgwtAMqQHjwNZIfRXX/9\n9V0tW69yww03dDXrmjhdHT9+PKxHScGsN5Kl0xYuXNjVsoNPo94safjyyy93tSj1+8ADD4SPj1J+\nV199ddj7iU98oqtFhz22FicFs/TgoUOHulqW2ByysimSfT4OgQTgjGRoAVCGoQVAGYYWAGUIYsyw\n6IvMT37yk2HvM88809Uuu+yysPf73//+yV0YzKAhK4WGyIJKUbAgCxtE9SwstW3btq523333dbXd\nu3eHj7/88su72hVXXBH2rl27NqxHovDKVIQrouBI9vjo9YYELjLutAAow9ACoAxDC4AyDC0AyjC0\nAChDenCGvfbaa13t3nvvHfnxd911V1hfsWLFiV4STKsoMZatUIpkvdFBhVG6bejrHTx4sKtNTEyE\nvffcc09X+9///tfVFixYED4+Sg++733vC3vnzZvX1bJDGYekByNZInBIenC6uNMCoAxDC4AyDC0A\nyjC0AChDEGOavP7662H9qquuGvk5fvOb33S1bMULVDLkXKUsQBCFGyYnJ8Pe6DysLMRw4MCBrvbU\nU0+FvQ899FBXi4IcH/jAB8LHX3nllV1t1apVYe+Qn3f27P5Pe3YuWRSkyIIrQz6fqVjZFHGnBUAZ\nhhYAZRhaAJRhaAFQhqEFQBnSg9PkjjvuCOtbtmwZ+Tk++tGPdrUsdQWVZP+Oo3p2sOOQ9VBHjx7t\navv27Qt7o/TgSy+9FPZG69OihO+ll14aPj5KDy5fvjzsjVJ+WUIvqkdrr1qLk4bZ+zhkFZT0IABn\nPUMLgDIMLQDKMLQAKEMQYwps3ry5q33ve9+b+QuBIoZ8ST+kNwttROuOonBGa61t3769q+3Zsyfs\nXbNmTVdbt25dV7vsssvCxy9ZsqSrZYGJKBxx6NChsDdaWxUFLlqLwxVD1jjNNHdaAJRhaAFQhqEF\nQBmGFgBlGFoAlCE9OAXuv//+rrZ///6RH58li6JD3+BMFq1xylY+RYc4ZonAqDd73uiwxEWLFoW9\nUXpw2bJlXW316tXh46OUXnSIZGtxIjCqtdba3Llzu1p2CGS11XDutAAow9ACoAxDC4AyDC0AyhDE\nmGFXX311V/v73/8e9gpicLaJ1gRlq4Oi9UPZqqJoXdK8efPC3ujsqw0bNoS90e/okPOposDEkHVL\n2flW0Tqr7H2Mghinw7qmjDstAMowtAAow9ACoAxDC4AyDC0Aypg1gymR0zeOwpmk1k6aovbu3Ttz\nfziSv1HZgY+RKCGXJfqydUeR6DmGpPGi9VKZ6HmzpGG11UyRsbGx8IdwpwVAGYYWAGUYWgCUYWgB\nUMZMBjEA4KS40wKgDEMLgDIMLQDKMLQAKMPQAqAMQwuAMgwtAMowtAAow9ACoAxDC4AyDC0AyjC0\nACjD0AKgDEMLgDIMLQDKMLQAKMPQAqAMQwuAMgwtAMowtAAow9ACoAxDC4AyDC0AyjC0ACjD0AKg\nDEMLgDIMLQDKMLQAKMPQAqAMQwuAMmbP4Gu9PYOvxdlr1qm+gLPBjh07/D4zrcbHx8PfZXdaAJRh\naAFQhqEFQBkz+Z0WwBll1qz+a5e3356er/vOOae/x3jrrbfC3ugaomutyJ0WAGUYWgCUYWgBUIah\nBUAZhhYAZUgPAiUNSchF9Sx5N0T0vFHKb0ii8I033gjrb7755ki11lqbM2dOV8vem+h6h/Rmpi1F\nOS3PCgDTwNACoAxDC4AyDC0AyhDEmAK//e1vu9qhQ4fC3kceeaSr/fznPx/5tb773e+G9Wuuuaar\nffzjHx/5eeF0lX2hP2SFUhRuyIIYx48fP+lrG/W1jhw50tWysEPUO3/+/LB3yZIlXW327PjPfRTm\nGBK4GGIqVkm50wKgDEMLgDIMLQDKMLQAKMPQAqCMWdO1aiMwYy80Xb72ta+F9Z/97GczfCW9TZs2\ndbV//etfYe+yZcum+3JOpTPjpLvT3I4dO2buD0fyNypKvWUrkKLk3a5du8LenTt3drUsUbht27au\ntn///q6WpeYWLVrU1RYsWBD2rlmzpquNj4+P3LtixYqwN5Kthxp1bVVWHzJvxsfHwzfNnRYAZRha\nAJRhaAFQhqEFQBnWOCWi0MVUBC6uuOKKrvb5z38+7N28eXNX+9WvfhX2PvXUU13t97//fdj75S9/\n+f93iXDKRF/UZ+GKo0ePdrV9+/aFvS+++GJXe/rpp8PeV199tasdPHgw7J2YmOhq8+bN62pZWCH6\nebPVTFHvqlWrwt4ozBGFPlqLQxeTk5Mj907Fmq0h653caQFQhqEFQBmGFgBlGFoAlGFoAVDGWZ8e\njFJFrbX2i1/8YuTn+OAHP9jV/vrXv4a9Cxcu7Gpz584Ne6OkznPPPRf2PvDAA11tz549YS+catmh\niJFjx46F9SjRl/2bf/bZZ7tatG6ptTjpl60+W79+fVcbGxvralu3bg0fH6UPs7RklGo8fPhw2Bv9\nDNkqqjlz5oz0+Nbiv0nZZzldh0u60wKgDEMLgDIMLQDKMLQAKOOsD2JkX9xG60aiwEVrrd19991d\nbfHixSd3Ya21O++8s6v997//Hfnxn/nMZ076GmBUU3E235CAxrnnntvVsnVA0bqjJUuWhL0XXHBB\nV8vOrYrOqNq9e3dXi1ZOtdbali1butqOHTvC3tWrV3e17Nyr6H3MQhuRLLQRnUuWBUei0Fm2omoI\nd1oAlGFoAVCGoQVAGYYWAGUYWgCUcdanB6+88sqwHqUKs3VL0YFrUyFaJZWttIFTbchBfllvVM9+\n76KEW3bQ4caNG7vaeeedF/ZeeOGFXS062LG11g4dOtTVonVLWRovqmepxuXLl3e1KKHXWmu7du3q\namvWrAl7Z8/ux0C2bmnIZzxd3GkBUIahBUAZhhYAZRhaAJRx1gcxMtn5OdPhrrvuCuuPP/74yM9x\n7bXXdrWLL774hK8Jpkr0pX72hf6QoFF0DlQW2ohWIJ1//vlh79KlS7tatL6otdb27dvX1Z588smu\ntm3btvDx0Zle69atC3s3bNjQ1bLgSfTeRKGR1uLPIlu3FD1vJlqzNRXcaQFQhqEFQBmGFgBlGFoA\nlGFoAVCG9OAMe/TRR7vaV7/61bA3Ojhu7dq1Ye9tt93W1YYkfWC6DDkcMloplCUNo0MNV65cGfaO\njY2N9FrZ805MTIS999xzT1d78MEHu9rLL78cPn79+vVd7dJLLw17N23a1NWy9GB0OGT29yA7SDIS\npTOzgzulBwE46xlaAJRhaAFQhqEFQBmCGDPsoYce6mpR4CJz0003hfV3vetdJ3xNMJ2iIEX25f8b\nb7zR1bKznSLZl/9RGGRycjLsjYIYTzzxRNj7yCOPdLWnn366q2VnWUWrpD72sY+FvatWrQrrkb17\n93a17Oyt6Pyu7POJPsss0DJd3GkBUIahBUAZhhYAZRhaAJRhaAFQhvTgNLnxxhvD+u9+97uRn+Ob\n3/xmV/v2t799wtcEp0KURMtWMw1JCi5evLirZSuFolTi66+/Hvbu2bOnq913331h7/bt27tatFpp\nwYIF4eOjwxaz1UxRSm/Xrl1h77x587patk4r+nyywzijz23Iuqbscx/CnRYAZRhaAJRhaAFQhqEF\nQBmCGFPg4MGDXe0vf/lL2HvkyJGuFq1yaa21W2+9tatF59nA6Sz68j0LBURhg6w3Oh8qCwXs37+/\nq+3evTvsfe6557razp07R76GKEixfPny8PEXXnhhWI9EAZGolr1eFoKI6tnfmSGhmula7+ROC4Ay\nDC0AyjC0ACjD0AKgDEMLgDKkB6fAdddd19UmJiZGfvw3vvGNsL5ixYoTviY4nQ1JsmWrnaJVRZko\ntRsdlNhavJopS9MtXbq0qy1btqyrnXfeeeHjL7nkkq4WJRJba+3QoUNdLVpP1Vprr732WleLDnts\nLX5vssRmtCYrW1E1FSubIu60ACjD0AKgDEMLgDIMLQDKEMQY4JFHHgnr995778jP8bnPfa6r3Xzz\nzSd6SVBSFq6IQghZKCCqZ88brVrbunVr2Ltt27aulp0vtXr16q62cuXKrpatcYrCVlnoIwpiZKGN\naDVcFoyIVl9Fr5VdW7TaKeudCu60ACjD0AKgDEMLgDIMLQDKEMRIHD58uKvdcsstYW/2JW3k/e9/\nf1dzRhZnsigwEW1WaK2148ePd7XsXKYodBH93rbW2quvvjpyb/T7nG3fWLt2bVcbHx/vahs3bgwf\nP3/+/K529OjRsDeqR2d3tRZv6sjCFdH7kIU2ou0Z2UaMLEATGbI9w50WAGUYWgCUYWgBUIahBUAZ\nhhYAZUgPJm6//fau9o9//GPkx994441h3comzjZRyi9Li0UpvWxNUHSWVHZG1osvvtjVnn766bA3\nSjC+4x3vCHvXrVvX1S666KKulp2NF703WRo5SgRmKczovcnSg1k9Ep3JFb1frUkPAoChBUAdhhYA\nZRhaAJQhiJG49dZbT+rxP/7xj8O6lU2cbYZ8yZ4FC0btnZiYCHu3b9/e1aJwRmvxeVjLli0Le8fG\nxkbujezfv3+kWmvxyqYsBBGdFfbKK6+EvVFoIwpctBavbMo+3yHnnQ3hTguAMgwtAMowtAAow9AC\noAxDC4AypAenycGDB8P6VKRnItH6m3PPPTfsjdbiZAfPRbLD82677baRnyOSXW+U5JwzZ85JvRYz\nJ/r3ln3Wkaz3wIEDXW3fvn1h78svv9zVooMhW2tt8eLFXS37Nx+93q5du0Z+rSjll/0uRgdORsm/\n1uIVVdnPEKUSN23aFPZGB3Jmn890/a1zpwVAGYYWAGUYWgCUYWgBUIYgxjSJztmZTjfddFNXy84A\nir4o/ulPfzrl1zRVovfyK1/5yim4Ek5EtOYnW9cUfXmfnacVnTuVrWbauXNnV5ucnAx79+zZ09U2\nb94c9r700ktdLVpflK1mis6yioIRrbW2YcOGrjYkIJKdb7VmzZquFoVRWovXO2WhqOizHHLGVsad\nFgBlGFoAlGFoAVCGoQVAGYYWAGVIDyZuuOGGrnbHHXecgisZze233z4tzztkbUvkS1/6Ulj/8Ic/\nPPJzfOQjHxm5l9NPlB6cihU/0eqy8fHxsHf16tVdLVuBFCX9HnzwwbA3Wtf2+uuvd7X58+eHj49S\neu95z3vC3ugAxuxQ2ag3SzRfcsklI/dGv/vZIZBRXXoQgLOKoQVAGYYWAGUYWgCUMWsqvhgb0Yy9\n0HT59a9/HdajdTJDPP7442H9ZFcrfetb3wrrGzduHPk5Pv3pT3e16Evt00j8rTBTaseOHSf1+5x9\neR8Ff7I1QdEKo927d4e9zz//fFd74YUXwt5ozdmWLVvC3ueeey6s/7+iVUmtxavWVqxYEfZG650W\nLlwY9q5fv76rrVy5MuxdunTpyM8b1bMwyJDAVmR8fDz8R+JOC4AyDC0AyjC0ACjD0AKgDEMLgDKk\nBznTSA/OgJNND2ZrnKJ61hul1o4cORL2Rkm27HDJ6DmigyFbi1c2DVlfFP1sWeouWgUVrbJqLU5c\nZu9j9HrZIZ3R62XXmyVERyU9CEB5hhYAZRhaAJRhaAFQhvO0gBmXhSCiAMCQL/qXLFkS9kbBhOx5\no2tYu3btyL1RkOPo0aPh4yPZdWXnf0WOHz/e1bIwSPQ+ZiGK6DlONnAxlDstAMowtAAow9ACoAxD\nC4AyDC0AypAeBGbckMRZtlIoOgQyO5AwSitmabzo2rIVSJOTkyO91pCkYvbzDnneaOVTlCjMnjcz\n00nBiDstAMowtAAow9ACoAxDC4AyBDGAkqLAwrFjx8LeqD4kgJCFI07W7Nn9n+As7BBdb/YzVAtX\nDOFOC4AyDC0AyjC0ACjD0AKgDEMLgDKkB4HTWpZui+rTlfI72YTdkDRfdlhjtEoq6z2TudMCoAxD\nC4AyDC0AyjC0AChj1tn4RR4ANbnTAqAMQwuAMgwtAMowtAAow9ACoAxDC4AyDC0AyjC0ACjD0AKg\nDEMLgDIMLQDKMLQAKMPQAqAMQwuAMgwtAMowtAAow9ACoAxDC4AyDC0AyjC0ACjD0AKgDEMLgDIM\nLQDK+D/00tnU8yfEwwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f88476905c0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Training one Autoencoder at a time in a single graph"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Another approach is to use a single graph. To do this, we create the graph for the full Stacked Autoencoder, but then we also add operations to train each Autoencoder independently: phase 1 trains the bottom and top layer (ie. the first Autoencoder) and phase 2 trains the two middle layers (ie. the second Autoencoder)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 150  # codings\n",
    "n_hidden3 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01\n",
    "l2_reg = 0.0001\n",
    "\n",
    "activation = tf.nn.elu\n",
    "regularizer = tf.contrib.layers.l2_regularizer(l2_reg)\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "\n",
    "weights1_init = initializer([n_inputs, n_hidden1])\n",
    "weights2_init = initializer([n_hidden1, n_hidden2])\n",
    "weights3_init = initializer([n_hidden2, n_hidden3])\n",
    "weights4_init = initializer([n_hidden3, n_outputs])\n",
    "\n",
    "weights1 = tf.Variable(weights1_init, dtype=tf.float32, name=\"weights1\")\n",
    "weights2 = tf.Variable(weights2_init, dtype=tf.float32, name=\"weights2\")\n",
    "weights3 = tf.Variable(weights3_init, dtype=tf.float32, name=\"weights3\")\n",
    "weights4 = tf.Variable(weights4_init, dtype=tf.float32, name=\"weights4\")\n",
    "\n",
    "biases1 = tf.Variable(tf.zeros(n_hidden1), name=\"biases1\")\n",
    "biases2 = tf.Variable(tf.zeros(n_hidden2), name=\"biases2\")\n",
    "biases3 = tf.Variable(tf.zeros(n_hidden3), name=\"biases3\")\n",
    "biases4 = tf.Variable(tf.zeros(n_outputs), name=\"biases4\")\n",
    "\n",
    "hidden1 = activation(tf.matmul(X, weights1) + biases1)\n",
    "hidden2 = activation(tf.matmul(hidden1, weights2) + biases2)\n",
    "hidden3 = activation(tf.matmul(hidden2, weights3) + biases3)\n",
    "outputs = tf.matmul(hidden3, weights4) + biases4\n",
    "\n",
    "\n",
    "with tf.name_scope(\"phase1\"):\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "    phase1_outputs = tf.matmul(hidden1, weights4) + biases4  # bypass hidden2 and hidden3\n",
    "    phase1_mse = tf.reduce_mean(tf.square(phase1_outputs - X))\n",
    "    phase1_reg_loss = regularizer(weights1) + regularizer(weights4)\n",
    "    phase1_loss = phase1_mse + phase1_reg_loss\n",
    "    phase1_training_op = optimizer.minimize(phase1_loss)\n",
    "\n",
    "with tf.name_scope(\"phase2\"):\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "    phase2_mse = tf.reduce_mean(tf.square(hidden3 - hidden1))\n",
    "    phase2_reg_loss = regularizer(weights2) + regularizer(weights3)\n",
    "    phase2_loss = phase2_mse + phase2_reg_loss\n",
    "    phase2_training_op = optimizer.minimize(phase2_loss, var_list=[weights2, biases2, weights3, biases3]) # freeze hidden1\n",
    "    \n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training phase #1\n",
      "0 Train MSE: 0.00777324\n",
      "1 Train MSE: 0.0075254\n",
      "2 Train MSE: 0.00773622\n",
      "3 Train MSE: 0.00787562\n",
      "Training phase #2\n",
      "0 Train MSE: 0.00219898\n",
      "1 Train MSE: 0.0025481\n",
      "2 Train MSE: 0.00238456\n",
      "3 Train MSE: 0.00263778\n",
      "Test MSE: 0.00289071\n"
     ]
    }
   ],
   "source": [
    "training_ops = [phase1_training_op, phase2_training_op]\n",
    "mses = [phase1_mse, phase2_mse]\n",
    "n_epochs = [4, 4]\n",
    "batch_sizes = [150, 150]\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for phase in range(2):\n",
    "        print(\"Training phase #{}\".format(phase + 1))\n",
    "        for epoch in range(n_epochs[phase]):\n",
    "            n_batches = mnist.train.num_examples // batch_sizes[phase]\n",
    "            for iteration in range(n_batches):\n",
    "                print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "                sys.stdout.flush()\n",
    "                X_batch, y_batch = mnist.train.next_batch(batch_sizes[phase])\n",
    "                sess.run(training_ops[phase], feed_dict={X: X_batch})\n",
    "            mse_train = mses[phase].eval(feed_dict={X: X_batch})\n",
    "            print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_train)\n",
    "            saver.save(sess, \"./my_model_one_at_a_time.ckpt\")\n",
    "    mse_test = mses[phase].eval(feed_dict={X: mnist.test.images})\n",
    "    print(\"Test MSE:\", mse_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa0AAAFxCAYAAADAqvdjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGzZJREFUeJzt3WmIlnXbx/G/6zib2mhONqSmVo6ttlpKlpBBQlERtECL\nBUVQUFBREFRvelm+iYr2DYK4U6J9U9upcGnPFk3GSbPJGWfVpu7Xz3P8juc5/16Lc8x8Py8Pjuuc\n87qm8bhPrt99/Ef9+++/CQCACEYf6BsAAKAohhYAIAyGFgAgDIYWACAMhhYAIAyGFgAgDIYWACAM\nhhYAIAyGFgAgjLFV/Fms3kA1jDrQNzASdHR08PeMimpqapJ/yzxpAQDCYGgBAMJgaAEAwqjmd1oA\nMKyMGmW/dlE1T6mnbPzzzz+ynnMP0fCkBQAIg6EFAAiDoQUACIOhBQAIg6EFAAiD9CCAYS8n5afq\no0eX9r/vc1J+g4ODha/7999/y/q4ceMKX6NU3nsr9TPz8KQFAAiDoQUACIOhBQAIg6EFAAiDIAaA\nkNQKpJxwhRcgGDNmjKl5gQfVq67r/SzFew/qHtTPTyml3t5eU8sJRtTU1Mh6zmee05uDJy0AQBgM\nLQBAGAwtAEAYDC0AQBgMLQBAGKQHAQwbOek0L3mnUm/79u0r/PNU78DAgHy9WrfkJfcmTJhgav39\n/YWv61HvNydZWc1DL1PiSQsAEAhDCwAQBkMLABAGQwsAEAZBDABDmrd+SIUFvGCCCjfkhCu8cIS6\nNxVWGD9+vHy9eg9eWKGnp8fU6urqZG/O2qqc9VDq/XrXLUfoQt5DRa4KAEAFMLQAAGEwtAAAYTC0\nAABhMLQAAGGQHgRQdV6yTNX37t0re1Xqra+vT/a2t7ebmpfoU4m8nPSf16uoNN7g4KDs9RKMRa/r\nUSufvGRlTnowJxmZc788aQEAwmBoAQDCYGgBAMJgaAEAwhhRQYxPP/3U1FauXCl7W1paTK22tlb2\nXnXVVabW1NQke706MFzlnE+lzp3yghgqXLF58+bCvd4ZV+pvf/LkybJ32rRpplZfX29q6iyslFKa\nOHGiqXnvV31m3toq1ev9+zVp0iRT8+5XhUG8e1CroHLO+fLwpAUACIOhBQAIg6EFAAiDoQUACIOh\nBQAIY1SlDuoSqvaDPEcddZSpeWmjUqlETkopLVy4sCI/r1JmzZplanfeeafsnTFjRoXvphB7+h3K\nrqOjw/w9q4MHU9LrfHp7e2WvSqLt2LFD9v7yyy+m9vXXX8telR70VgeNHWtD1WplVEopdXV1Fer1\nDmvcs2ePqXlpPPVvikofej/PS0CedtppprZgwQLZ29DQYGre2in134N3uKTS1NQk/4PiSQsAEAZD\nCwAQBkMLABAGQwsAEMaIWuO0atUqU9uwYYPsPfroo03tm2++kb2fffaZqa1evVr2vvnmm6Z2+OGH\ny95ff/1V1otSXyinlNL06dNNbdu2bYWvq8IZKaV0xx13FL4Ghh8v1KW+kM85P8lbKaRCAXPnzpW9\nKgDgnU+lwhEqyJFS8XVJf/31l3z9H3/8YWreeVzqc1CvTymlzs5OU5s6darsVWurTj31VNmr/k3x\nVjOpAI4XaOE8LQDAsMTQAgCEwdACAITB0AIAhMHQAgCEMaLWOFWTt4ply5YtpualB9WamhxeCkml\nB717UOmkl19+WfZecMEFGXdXMaxxqgK1xsmj0oNeikytBOrp6ZG9KuXnrYdS1/BWIO3evdvUuru7\nZa9al6RSid57UCuq1CGSKekDZD/88EPZq9LLJ598suw966yzTO3cc8+VvY2NjabmzRD1O/ZWPims\ncQIAhMfQAgCEwdACAITB0AIAhDGi1jhVk7d6Zt68eYWv0draWq7b+R/U2qldu3bJXnXWzrJly8p+\nT4jPO09L1b3VP6ru9apQgHcPRX9WSint3bu38HXVeihVU+uevJ+l1lOlpIMN77//vuxVwRH1eaWU\n0pFHHmlqOauZvCBGTugiB09aAIAwGFoAgDAYWgCAMBhaAIAwGFoAgDBIDw5j3uqYCy+80NRUKiil\nlB588EFTU4fcAR7135ZK2KWkU3ref5vq4EAv9aZ6vUSfujcvDVw0Iecl7FSiz/tsfvrpJ1P74Ycf\nZO+kSZNMzVvVpta6eQfIVioRmIMnLQBAGAwtAEAYDC0AQBgMLQBAGAQxhrGnnnpK1n///XdTmzJl\niuydOXNmOW8Jw1jO2XxeuEIFMbwv/9U1vACBCl14Z3qpe1DrllIq/p6996DuQZ1hl1JKzzzzjKlt\n375d9p544ommtmTJEtmrzu/ywiA5a5wqhSctAEAYDC0AQBgMLQBAGAwtAEAYDC0AQBikB4eJn3/+\n2dRuvfXWwq//5JNPZP2QQw7Z73sCcql0mpfyU6k1L5Wo0n9eQs5LCip1dXWm1tfXZ2o1NTXy9Sqp\n+P3338vetWvXmpr3Hs455xxTa25ulr0qceklAtX95hy8WQ48aQEAwmBoAQDCYGgBAMJgaAEAwiCI\nMUy88sorpuadF3TJJZeY2uzZs8t+T4DH+6JfBQtU2CElvRrJC2Ko87C8AIG6B++66sw6dV0viKFW\nNn311VeyVwVETjnlFNl7wgknmJp3JlhOkII1TgAAZGBoAQDCYGgBAMJgaAEAwiCIEYwXrnj55ZdN\nzfvy9/777zc17/9ZDxxo3hf9apNDzjldntGj7f+W7+7uLnzdyZMnm5q31ePXX381tS+++EL2trS0\nmNrSpUtlr9p+MX78eNk7MDBgauozSEn/O+G9N3UN7/eTgyctAEAYDC0AQBgMLQBAGAwtAEAYDC0A\nQBikB4N5/PHHZf2DDz4wtcsvv1z2srIJB5qXCFRJNC/1pq7hpWtV0tBLvamE3Lhx42RvY2OjqTU0\nNJja1q1b5etXrVplat55WuqMrMWLF8tetbLJOydMrcNSn1dKOv3nJTPVdcuBJy0AQBgMLQBAGAwt\nAEAYDC0AQBgEMYawDRs2mNpNN90ke9XqmPvuu6/s9wSUQ85apZzAhLeOTK0U8sIGKkBQX19f+B46\nOztN7eOPP5avX7dunal5514tW7bM1KZNmyZ7VSDF+xxV0MVb46R+b16oJuecraw1W4U7AQA4wBha\nAIAwGFoAgDAYWgCAMBhaAIAwSA8OAX19fbJ+2WWXmZq3GuWKK64wNdY1YSjIOQxQJc68JJtKw3np\nQXVdbzWTujfvHtR1N23aZGqvvfaafL06XPLSSy+Vvccdd5ysK+qz8f6dUauoPOqz8VZnqd5yHDbL\nkxYAIAyGFgAgDIYWACAMhhYAIAyCGFWmvpxcvny57P3hhx9MrbW1Vfbee++9pd0YUCFe6KIo74t+\nLxyhqGCCF2pSa428lU/bt283tRdeeMHUVq9eLV+/ZMkSUzvvvPNk78SJE03New979uwxtZqaGtmr\nPsecz9y7BxVS8dY15ax84kkLABAGQwsAEAZDCwAQBkMLABAGQwsAEAbpwSrr6OgwtTVr1hR+/bPP\nPivrTU1N+3tLQEWpxJiXOFO81T979+41NZX8Symluro6U/MScup+BwYGZO/69etNbe3ataY2c+ZM\n+forr7zS1ObMmSN71efQ398ve9X79RJ66hreZ64+My9ZqX5eTkrQw5MWACAMhhYAIAyGFgAgDIYW\nACAMghgV0tnZKesLFy4sfI3nnnvO1BYsWLDf9wQcCCrYkHM+lUeFLrxwRU9Pj6l5YQO1dqqtrU32\nvvrqq6a2Y8cOU7vmmmvk60899VRTUyGKlHR4xXsPamWTd56Wuq5ae5WS/l16v7OcNU45eNICAITB\n0AIAhMHQAgCEwdACAITB0AIAhEF6sEKefPJJWf/ll18KX2Px4sWmVo70DVBNpabIvNVMKuHmrRRS\n1/BWSamk4KpVq2Tvf/7zH1ObMGGCqc2fP1++furUqabmvV+1bsl7vzlJQ5WW9BKB6hreIZ+V+reK\nJy0AQBgMLQBAGAwtAEAYDC0AQBgEMcpg8+bNpnbPPfdU/0aAIajUM5S8M6NUACAnFNDd3S17N23a\nZGobN26UvWpd25IlS0ztsMMOk6/v7e2VdUW9B+/9emuYFPX78UIbireSq1J40gIAhMHQAgCEwdAC\nAITB0AIAhMHQAgCEQXqwDD744ANT6+rqKvz61tZWWa+trd3vewKGCy+dpuo5vV6qUaUV1WqmlFI6\n99xzTW3RokWm1tzcLF+vUnreQZbqvsrxflUqcSivi+NJCwAQBkMLABAGQwsAEAZDCwAQBkGMKjvj\njDNM7e2335a9BDGAPF6AQK01GhgYkL3Tpk0ztXPOOUf2NjU1mdoxxxxjao2NjYXvywtXlBqY8K7r\nrYIaqnjSAgCEwdACAITB0AIAhMHQAgCEwdACAIQxqtQD2jJU7QdhRBu6+2eGkY6OjvB/zzlrnLyE\nnVrvNG7cOFPzUn45BztGS/mVqqmpSX5oPGkBAMJgaAEAwmBoAQDCYGgBAMKoZhADAICS8KQFAAiD\noQUACIOhBQAIg6EFAAiDoQUACIOhBQAIg6EFAAiDoQUACIOhBQAIg6EFAAiDoQUACIOhBQAIg6EF\nAAiDoQUACIOhBQAIg6EFAAiDoQUACIOhBQAIg6EFAAiDoQUACIOhBQAIg6EFAAiDoQUACIOhBQAI\ng6EFAAiDoQUACIOhBQAIg6EFAAiDoQUACIOhBQAIY2wVf9a/VfxZGLlGHegbGAna2tr4e0ZFtbS0\nyL9lnrQAAGEwtAAAYTC0AABhVPM7LQDINmpU6V9T/vtvaV/B5dzDP//8U/jnjx5tnxvU673enHvI\nef1QNjzeBQBgRGBoAQDCYGgBAMJgaAEAwmBoAQDCID0IYEgrNfnn8VJ6KinoJe9yeovew759+2Rv\nXV2dqfX29pZ8X6rXS0uOGTNG1ote1/vMc/CkBQAIg6EFAAiDoQUACIOhBQAIgyBGGTz//POm1tPT\nI3u//PJLU3v00UcL/6y7775b1pcuXWpqZ511VuHrAtXkfdGfE7pQvYODg7J37969he/h77//NrXu\n7m7ZOzAwYGp//fWXqXn/HtTX15vauHHjZO+kSZNMraamRvaqa4wdq/+5V721tbWyt1KhmBw8aQEA\nwmBoAQDCYGgBAMJgaAEAwmBoAQDCGFXFNMiBj52U6MYbb5T1Rx55pMp3Ys2fP9/UPvzwQ9mrUkjD\nSOknBuL/1dbWVtLfc0560Ps3SiXyvJTetm3bTO3nn3+WvRs2bDC1b7/9Vvbu2rXL1Do6Okxt6tSp\n8vVz5841tXnz5slelRA+/PDDZe/BBx9sat4aJ/X55qy48n6XOWuc1D20tLTIC/OkBQAIg6EFAAiD\noQUACIOhBQAIgzVODhW6KEfgYsGCBaZ28cUXy97Nmzeb2tNPPy171RfFL730kuy99tpr/69bBIaU\nnNVM6m8mpZS+//57U1u7dq3sXb9+val5a41aWlpMbdmyZaa2Z88e+fq2tjZT+/PPP2Xvpk2bTG3W\nrFmyV62H8j5HFV7xVkn19/ebmhfwUNfw7iHnDDKetAAAYTC0AABhMLQAAGEwtAAAYTC0AABhjPj0\n4G+//Sbrjz32WOFrnHLKKab2xhtvyN66ujpTGz9+vOxVSZuffvpJ9n700UemplbMANXmrflR1Jqf\nffv2yV512KK38kkl8lTCLqWUli9fbmqnn3667D3zzDNNbcKECab26aefyte/8MILpvbHH3/I3qOO\nOsrU+vr6ZG97e7upTZw4UfaqRKCqedShmSnpxKV3EGUOnrQAAGEwtAAAYTC0AABhMLQAAGGM+CCG\nF1ZQX+iqwEVKKb3zzjum1tDQUNqNpZSeeuopU/v8888Lv/6CCy4o+R6AUuWc16RWM3krftQX/V5o\no7m52dTUmVMppbRo0SJT89YlqXOyuru7TW3nzp3y9d98802ha6akgyPevzMq8OCtUKqpqTG13t5e\n2atCF174RQVwcs708vCkBQAIg6EFAAiDoQUACIOhBQAIg6EFAAhjxKcHTzzxRFlXqUJv3ZJ3QFyp\n1Copla4ChrKcZJhKnHkHEh566KGm5q0Jmjt3rql5f7cqveel3gYGBkyto6PD1DZs2CBfr67b2Ngo\ne2fOnGlqamWUV/feg/rMvc9GvV/v96uShqxxAgCMKAwtAEAYDC0AQBgMLQBAGCM+iOGZNGlS1X7W\ns88+K+sbN24sfI1ly5aZ2pw5c/b7noBcOYGLMWPGyLoKC3hhAxWMUiuJUkqpp6fH1Lz7VSumvABB\nV1eXqa1bt87UvvzyS/l6dV0VGklJr6LyzrJSq6S8f9PUPeQEvryAmuKt7/JCIrK3cCcAAAcYQwsA\nEAZDCwAQBkMLABAGQwsAEAbpwSpbv369qV1//fWyV61MmT59uuxduXKlqXnrb4BKyEmAeVSq0Esa\nqvSf+pvJ+Vkp6fexe/du2asSviop6B2qqP6eTzvtNNk7Y8YMU/MOdmxvbzc1bzVTTupTXcP7vau6\nd785eNICAITB0AIAhMHQAgCEwdACAIRBEKPKPvnkE1PL+fL4hhtukPUjjzxyv+8JKAfvC311XpOq\npaTDEd7qHxVu2Ldvn+xV1/DWGnV2dpratm3bZO/bb79tau+9956peWdkLVy40NRaW1tlrwpWqXP/\nUso7E0x9jjkhFW+Nk/pdePeQEwbhSQsAEAZDCwAQBkMLABAGQwsAEAZDCwAQBunBClmxYoWsv/ji\ni4Wvccstt5ja7bffvt/3BFSSlwhUyTCvN4dKp3mHIqrDIb2koUoPfvTRR7J3zZo1pqbe7+LFi+Xr\n58+fb2peqnHnzp2mVl9fL3unTJliat7Bjiop6KX8chKBqu4lQXP+e+BJCwAQBkMLABAGQwsAEAZD\nCwAQBkGMMuju7ja1119/Xfb29/ebWnNzs+y96667TM1bmQIcaDmreLwv3tUX/V6AQH3RX1dXV/jn\ntbW1yd7vvvvO1NQZWSnpgMexxx5rascff7x8vQpoeH/jXV1dpuaFNrxAiqL+TfICHmPH2pFRjnAF\na5wAAMMSQwsAEAZDCwAQBkMLABAGQwsAEAbpwTK45JJLTE2tXPHcfPPNst7U1LTf9wQMZTkHRnpJ\nuNraWlNTByWmpBNy7e3tsvezzz4ztS+++EL2zp4929TmzZtnascdd5x8fUNDg6l566VUr/fZqGTl\n9u3bZe/kyZNNLSfd6SU2BwcHZb1UPGkBAMJgaAEAwmBoAQDCYGgBAMIgiJHBW+WiztTxXHTRRaZ2\n66237u8tAUOe+lI/54v+nJVCOUGMjRs3yl51dlZPT4/sVSEEtbJJhShSSqm3t9fU1PtKSa+M2rNn\nj+xVn0NjY6PsVYEJ7/ejVjapz9a7Rs7ZWx6etAAAYTC0AABhMLQAAGEwtAAAYRDEcPT19ZnanXfe\nKXu9836Uk046ydQ4IwvDmfpCPuf8JC+YoLZBDAwMyN7169eb2rfffit7t2zZYmoHH3yw7F26dKmp\nzZo1y9TU9o6U9Gfjvd+cYIMKtHhnbyk5Z5h595sT8MjBkxYAIAyGFgAgDIYWACAMhhYAIAyGFgAg\nDNKDjocfftjU3n333cKvX7FihayzsgkjjVr946XexowZY2peOk0lEFXyL6WU1q5da2pvvfWW7FX3\ntmTJEtl79tlnm1pLS4upealGlbDzVj6p+1KfV0o6/ewlNlVS0DvTS/V652Z5iclS8aQFAAiDoQUA\nCIOhBQAIg6EFAAiDIIbjrrvuKun1DzzwgKyzsgkjTc4aJ3UOlBc26O7uNjUv8LBt2zZTU2dZpZTS\nnDlzTE2tX0tJhy4mTJhgal5YQa2i8kIQ6rPxrqv+neno6JC9KrShap6ZM2fKuro373eZgyctAEAY\nDC0AQBgMLQBAGAwtAEAYDC0AQBikBytEJZtS8tfXlKqmpsbUvKSOSvV4qSvFSxatXLmy8DUU735V\nklMlqTA0qaSg93egVj7lXNdLyO3cudPUvPRgf3+/qXkpvR9//NHU1HtQKcGU9L8TXnpQ/Y1711X1\n3377Tfaq34X3+2lqajK1+vp62TtlypTC1805HJInLQBAGAwtAEAYDC0AQBgMLQBAGAQxKkStd6mk\nG264wdQOPfRQ2fv777+b2kMPPVT2eyoX9Vled911B+BOsD9yztNS1BlOHrVCKaWUpk6dampeuEKt\nfHriiSdk70EHHWRqf/75p6m1trbK16uzwnJWXHnvYceOHabW1dUleydOnGhqJ5xwguydPHmyqXnB\nLFX3fj/ee1Z40gIAhMHQAgCEwdACAITB0AIAhMHQAgCEQXrQccUVV5jak08+eQDupJiHH364ItdV\n6aacg9yuvvpqWT/99NMLX2PRokWFezH0qKSglxZT/715q4rUdRsbG2XvEUccYWpeok+lB9vb22Xv\nli1bTK2zs9PU1BqplHQyUq1KSkmn/Ly/RVX3VssdcsghhWoppXTYYYeZmlrXlFJKtbW1ppaTEvTw\npAUACIOhBQAIg6EFAAiDoQUACGNUOb4YK6hqP6hSnnnmGVnPWTOjbNy4UdZLXa102223yfrcuXML\nX+P88883tWnTpu33PVVB8YN5sN/a2toK/z2rs5K885NUEMM7O02th/L+Fr/++mtT8/7uenp6TM07\nb06d37V161ZT887IUmd6zZgxQ/aqVVTeWVbNzc2FflZKOkgxb9482atCIl5wRMk5N6ulpUU286QF\nAAiDoQUACIOhBQAIg6EFAAiDoQUACIP0IIYb0oNVUGp60KOSgt7BgeqQQW+Nk0oEeklD9W/i+PHj\nZa9K5Knremugdu/ebWpeOle9N2+NU01Njal567AU77pqdZZKfHq9OfOG9CAAIDyGFgAgDIYWACAM\nhhYAIAzO0wJQUTlfvg8ODpqaClykpL/o986MUiGErq4u2auCI966JLVaSb1++vTp8vUNDQ2m5r1f\nFY7wetVn3t/fL3vVOiwvPJPzu6xUyI8nLQBAGAwtAEAYDC0AQBgMLQBAGAwtAEAYpAcBDBkqyaZq\nKel0mkoUpqRTdl5CTtW9Qxx37dol6/+bt+oo577UNbz0oFol5X026ro56cEqrgJMKfGkBQAIhKEF\nAAiDoQUACIOhBQAIgyAGgCFDfamfE5jwemtra03NC0fkXFfdb06YRAUpvF51D+r8sZTywhXezxuq\neNICAITB0AIAhMHQAgCEwdACAITB0AIAhEF6EMCwoQ6RTEkn53LSg56iK4zUAY7lMBQOZaw2nrQA\nAGEwtAAAYTC0AABhMLQAAGGMGi5fzgEAhj+etAAAYTC0AABhMLQAAGEwtAAAYTC0AABhMLQAAGEw\ntAAAYTC0AABhMLQAAGEwtAAAYTC0AABhMLQAAGEwtAAAYTC0AABhMLQAAGEwtAAAYTC0AABhMLQA\nAGEwtAAAYTC0AABhMLQAAGEwtAAAYTC0AABh/BdWahcyEjHmHgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f883ffffba8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs, \"./my_model_one_at_a_time.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Cache the frozen layer outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training phase #1\n",
      "0 Train MSE: 0.00725573\n",
      "1 Train MSE: 0.00784995\n",
      "2 Train MSE: 0.00773007\n",
      "3 Train MSE: 0.00777166\n",
      "Training phase #2\n",
      "0 Train MSE: 0.002398\n",
      "1 Train MSE: 0.00257562\n",
      "2 Train MSE: 0.00279413\n",
      "3 Train MSE: 0.00285046\n",
      "Test MSE: 0.00302882\n"
     ]
    }
   ],
   "source": [
    "training_ops = [phase1_training_op, phase2_training_op, training_op]\n",
    "mses = [phase1_mse, phase2_mse, mse]\n",
    "n_epochs = [4, 4]\n",
    "batch_sizes = [150, 150]\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for phase in range(2):\n",
    "        print(\"Training phase #{}\".format(phase + 1))\n",
    "        if phase == 1:\n",
    "            mnist_hidden1 = hidden1.eval(feed_dict={X: mnist.train.images})\n",
    "        for epoch in range(n_epochs[phase]):\n",
    "            n_batches = mnist.train.num_examples // batch_sizes[phase]\n",
    "            for iteration in range(n_batches):\n",
    "                print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "                sys.stdout.flush()\n",
    "                if phase == 1:\n",
    "                    indices = rnd.permutation(len(mnist_hidden1))\n",
    "                    hidden1_batch = mnist_hidden1[indices[:batch_sizes[phase]]]\n",
    "                    feed_dict = {hidden1: hidden1_batch}\n",
    "                    sess.run(training_ops[phase], feed_dict=feed_dict)\n",
    "                else:\n",
    "                    X_batch, y_batch = mnist.train.next_batch(batch_sizes[phase])\n",
    "                    feed_dict = {X: X_batch}\n",
    "                    sess.run(training_ops[phase], feed_dict=feed_dict)\n",
    "            mse_train = mses[phase].eval(feed_dict=feed_dict)\n",
    "            print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_train)\n",
    "            saver.save(sess, \"./my_model_cache_frozen.ckpt\")\n",
    "    mse_test = mses[phase].eval(feed_dict={X: mnist.test.images})\n",
    "    print(\"Test MSE:\", mse_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa0AAAFxCAYAAADAqvdjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGvlJREFUeJzt3VuMnmX1N+C7QGmHUjoUCmWvlU0LB+yhbGIQIh4QJGqI\nUeImaCIx0UQSNZCYqCceKh4YNBpQ9EBjgpsQMVAkImCthJ0Khk3ZCwyp3dGZboD/0fcl/+9ei+99\n6My0i7muw5X1PvO8bztdefL+uu55b731VgOACvbZ0zcAAKMytAAow9ACoAxDC4AyDC0AyjC0ACjD\n0AKgDEMLgDIMLQDK2G8Wf5bVG8yGeXv6BuaCjRs3+n1ORFuG5s3z13Ko8fHx8EPzpAVAGYYWAGUY\nWgCUMZvfaQGUNOQ0jKg3e330Xdd0fP8VXePdcqKHJy0AyjC0ACjD0AKgDEMLgDIMLQDKkB4ESpqp\nlN6Q5N2uXbve7hb/rzfffDOsz58/v6u98cYbI/fuu+++YW90vzt37gx799mnf3YZ8jnONk9aAJRh\naAFQhqEFQBmGFgBlCGIAe7UhoYAoVNBaHG7IQgxRbxaO2G+//p/QKPAQ9WXXnY51S1nwY9R7GBJS\nmW2etAAow9ACoAxDC4AyDC0AyjC0AChDehDYq2VpuiEpuywpOGrv/vvvH/ZGa5wWLlzY1bI0X1TP\nEnpTU1NdbcGCBWHv5OTkyNeNko1DUoJZ70wdOulJC4AyDC0AyjC0ACjD0AKgDEEMoKQoxJB9+R+F\nDbJwRLQKavv27WFvtAIpOvcqe/2WLVtGvq8lS5Z0tWw91OLFi7vajh07wt4oeBK9h0z2mUefzZD1\nUhlPWgCUYWgBUIahBUAZhhYAZRhaAJQhPQjsNaIkWpY4i9Yabdu2LeyNrpEd7HjggQd2tWyNU1SP\n1jhF12yttaVLl3a17P0OSfQNSUAOSWEOWTuVfb67y5MWAGUYWgCUYWgBUIahBUAZcyqI8de//rWr\n3XDDDWHvUUcd1dXGxsbC3s985jNdLfqC9e3qMJdkYYNo1dCGDRvC3scee6yrTUxMhL2bN2/ualnY\nYPny5V0tC1JE/yYsW7asq0UrmFqLz8PKAhPR55CFM6IwSHam2KJFi7ra+Ph42DsktBHVs9DGkPO7\nPGkBUIahBUAZhhYAZRhaAJRhaAFQxrws+TEDZu0HZU466aSu9sQTT8zIz8rSQqtXr56RnzdT3vOe\n93S16667Luw99thjZ/huRjJ6DIl3bOPGjbv1+7x169awPjk52dWee+65sPfRRx/talFKsLXWXn31\n1a6WpfR27do18nWjVUWbNm3qalGisLV43VL0GbQWJwWjlGBrcXIv+l1urbUPfvCDXe2UU04Z+R6y\nJGhUzw6tjIyPj4e/y560ACjD0AKgDEMLgDIMLQDKmFNrnH7zm990tYceeijsjb6I/Oc//xn2rl27\ntqv99re/DXv/+Mc/drX3vve9Ye/69evD+qiyLz2POOKIrvb888+PfN3sC92vf/3rI1+Dd59sFU8U\nCsjOp4rWOEWrjlqLVw1l143uLQuhRUGIaNVRa629/PLLYf3/lYU+Nm7c2NWydUtR0Cnrffzxx7ta\ntvLplVde6Wonnnhi2BtdI7uHKGQyHTxpAVCGoQVAGYYWAGUYWgCUYWgBUMacWuM0m6ampsL6M888\n09Wy9ODTTz+9W/eQJami9GB2D9GherfeemvYe8UVVwy4uxljjdMs2N01TtH6o9Za27ZtW1fbsmXL\nyNfIUnrR72OWpouuka2ditKO0UGv//3vf8PXR9c97LDDwt6DDz64q915551h75o1a7raMcccE/Ze\nddVVXe3MM88Me6PPLFp71Vq8xmnIYY/WOAFQnqEFQBmGFgBlGFoAlDGn1jjNpuycm5UrV458jVWr\nVk3X7fwv0dqp1157Lew999xzu9qll1467ffE3JKtGIvOocvWOEVrgrJgWXSNLAwS1YecGRUFoLKA\nSHRf2cqoKPRx1113hb3RKqrFixeHvYceemhXy1YwRZ9NFq4YEroYwpMWAGUYWgCUYWgBUIahBUAZ\nhhYAZUgPvou9/vrrYf0jH/lIV8vSUd/73ve62tjY2O7dGHPKkBTZqGm81uIEYvb3OOrN0oNRAjFL\nO0Ypu+hQxOygxCH3FR3s+I9//CPsjdZDHX/88WFvtDYqSw9GK5uGHP6ZGfJ3xJMWAGUYWgCUYWgB\nUIahBUAZghjvYjfffHNYf/nll7vaIYccEvYed9xx03lLzEEzdWZf9OX9zp07R76HLLQRhQ2yIEZU\nz+4hEoUuXn311bD3d7/7XVe79957w97Vq1d3tbPPPjvsPeCAA97uFv+X6DPLQibWOAEw5xlaAJRh\naAFQhqEFQBmGFgBlSA++Szz11FNd7dprrx359ffff39YX758+Tu+J8hkq4qi9UFZCi1aVTQ1NRX2\nDknIRQczZgnIKGkY3cP8+fPD10fJuyeffDLsXbduXVfL3tcHPvCBrnbCCSeEvdHnmyUrh6QwR319\na/naqLB35E4A2MMMLQDKMLQAKMPQAqAMQYx3id///vddLVsnc+WVV3a1FStWTPs9QWbIeU3ZF/1R\nmCMLPES92fqh6OdlYZDodyw6/2vhwoXh67ds2dLV/vSnP4W9L730Ulc744wzwt7zzz+/qw0Jo0R/\nDpmZWteU8aQFQBmGFgBlGFoAlGFoAVCGIEYxWbji1ltv7WrR/+xvrbXvfOc7XS37UhpmwpAv77Mg\nRhTmWLRoUdgbBQuye4iCFNnv3Y4dO0Z6ffb7tX79+q72wgsvhL3RmXeXX3552HvkkUd2texMsOg9\nZBtLoo0WQ/4sp+NsNU9aAJRhaAFQhqEFQBmGFgBlGFoAlCE9WMxPfvKTsH7PPfd0tU9+8pNhr5VN\n7K2iRGCWOBsbG+tqWZItum6WkIvSitu3bw97R10l9eKLL4avv+2227padDZea61dcsklXe20004L\ne6Ok4JD3m6Udo883W8kV9UoPAjCnGFoAlGFoAVCGoQVAGYIYe7GHHnqoq33pS18Ke8fHx7vat7/9\n7Wm/J5ht2RlZQ1aPRUGKLEAQhQWy3uicrKmpqa4WBaVaa23NmjVdLVu3dN5553W1pUuXhr3RZxOt\na2otDkwMOT8s+2yy97G7PGkBUIahBUAZhhYAZRhaAJRhaAFQhvTgXmBycjKsf+ITn+hq2SqWq666\nqqtZ10Q127Zt62rZwY6RbE1QlIaLDmtsLU4rZmm6aAVSlPqN1jW1Fh8C+alPfSrsPfnkk0e+r+hz\nzFJ+UT277pADH4ccvDnkup60ACjD0AKgDEMLgDIMLQDKEMSYZdEXt5dddlnY++9//7urrVq1Kuz9\n1re+tXs3BjMk+jufffEerR8a8iX9kCBGttYouocsAPXss892tV/96ldd7b777gtff+6553a1j3/8\n42Hv4Ycf3tWyc76idUuZKJCShTaizyZbpxV9Ztmf5ZBztjxpAVCGoQVAGYYWAGUYWgCUYWgBUIb0\n4CzbsGFDV7v77rtHfv0tt9wS1rPD4GBPixJjQ9JiWRIuS7hFogMJo1Rja8NSbw8//HBXW7duXVdb\nuXJl+Pprrrmmq2UJ4SGi95sdprm7181En++QP/eMJy0AyjC0ACjD0AKgDEMLgDIEMWbIpk2bwvrq\n1atHvsbPf/7zrnb66ae/43uCvUX2hXwUFshWKEXXyMIVUWgjC3JE133iiSfC3rVr13a1rVu3drWL\nL744fH10Rla2Fila2ZR9NgsWLOhqQ9ZhDTHkz2c6eNICoAxDC4AyDC0AyjC0ACjD0AKgDOnBGXLT\nTTeF9aeffnrka1x44YVdbaYSQDBTor+zWXJvyIGRkV27do183Wwl0ZYtW7raPffcE/b+/e9/72qL\nFi3qalFKMOvN0nhRPTrAMZOl+aLPLPvziVZqDVmnNR08aQFQhqEFQBmGFgBlGFoAlCGIMQ2iFS/f\n/OY3Z/9GoLhohVF2DlQU0IjWF7UWBzEmJyfD3gcffLCrrVmzJuyN1jhddNFFXW18fDx8ffTesjBJ\n1JuFK4acZRXVs3uY7dBFeA97+gYAYFSGFgBlGFoAlGFoAVCGoQVAGdKD0yBa8bJ58+aRX79q1aqw\nPjY29o7vCfZm2WqmqJ6tW4p6d+zYEfZGScEsPRitKlq+fHnYe9lll3W1D33oQ10tW+O0u6uZontt\nLU4EZsm/qJ4lDfeGNXKetAAow9ACoAxDC4AyDC0AyhDEmGXnn39+V7vjjjvCXkEMyFcKRaGAIQGP\nLJhw5JFHdrXTTjst7D344IO72kknndTVonOzWouDGNu3bx+5NwtMDFm3tLuf42zzpAVAGYYWAGUY\nWgCUYWgBUIahBUAZ87L0yQyYtR/EnLbn401zwMaNG9+1v8/Rv4nZv5NRoi86gDFL80WHXmYJvZn6\nt3pvONgxMj4+Hn4Qe+fdAkDA0AKgDEMLgDIMLQDKmM0gBgDsFk9aAJRhaAFQhqEFQBmGFgBlGFoA\nlGFoAVCGoQVAGYYWAGUYWgCUYWgBUIahBUAZhhYAZRhaAJRhaAFQhqEFQBmGFgBlGFoAlGFoAVCG\noQVAGYYWAGUYWgCUYWgBUIahBUAZhhYAZRhaAJRhaAFQhqEFQBmGFgBlGFoAlGFoAVDGfrP4s96a\nxZ/F3DVvT9/AXDAxMeH3mRm1bNmy8HfZkxYAZRhaAJRhaAFQxmx+pwWw15s3r/8q5a23Rv8Kb3df\nz9vzpAVAGYYWAGUYWgCUYWgBUIahBUAZ0oPAnBSl/LL6zp07w94333xz5OtG9tmnf27IXr9jx46u\ntu+++478s+bPnx/Wo2TjkPcw2zxpAVCGoQVAGYYWAGUYWgCUIYgxDX7xi190tddffz3sfeCBB7ra\nj370o5F/1je+8Y2wfvHFF3e1iy66aOTrQjW7du3qalEworU4xBDVWmtt69atXe2ll14Kezdv3tzV\ntm3b1tWWLl0avn58fLyrHXDAAWFvFKQYGxsLexcuXNjVss8musYbb7wR9u4NAQ1PWgCUYWgBUIah\nBUAZhhYAZRhaAJQxbxYPJyt/CtoXv/jFsP7DH/5wlu+kd/LJJ3e1v/zlL2HvkiVLZvp29qQ9H2+a\nAyYmJvb473O0WilK87XW2qZNm7ra448/HvauW7euq913331h74svvtjVouTeggULwtcfc8wxXe38\n888Pe88555yutnLlyrD3wAMP7GpZWvKggw7qalEys7X4fWS9WQJxVMuWLQt/lz1pAVCGoQVAGYYW\nAGUYWgCUYY1TIgpdTEfg4vTTT+9qH/vYx8LeJ554oqv99Kc/DXv/9a9/dbVf//rXYe/nPve5t7tF\n2GOiNUFZWGxqaqqrvfLKK2HvI4880tWiwEVrrT366KMjXzcKUkThiGg1VGutPfbYY10tC4iceOKJ\nXS06j6u1OASR9UaBliw4El13+/btYe+QlU9DzgXzpAVAGYYWAGUYWgCUYWgBUIahBUAZcz49+Nxz\nz4X1H//4xyNf4+yzz+5qt99+e9gbHfC2//77h71RUufJJ58Me++9996u9tprr4W9sLeKkoLZ+qHo\n7/f69evD3qeffrqrRYc1ttbaqaee2tXOOuussPfyyy/vatHBjtkaqO9///td7YUXXgh7o1VUUS27\nhywRuN9+/RjIEpuTk5NdLUsJRqnE6Ge93c+LeNICoAxDC4AyDC0AyjC0AChjzgcxsrBC9MVgFLho\nrbU777yzq0Xn2Qx18803d7Vs9Uzkiiuu2O17gNkUnc2UrQl6/fXXu1oWNjj22GO72imnnBL2Hnfc\ncV0tOsuqtfhsupdeeqmrZQGqaP3aEUccEfZOTEx0tSFBjEWLFoW9kewzj9YtZSuYooDGkHVNGU9a\nAJRhaAFQhqEFQBmGFgBlGFoAlDHn04NnnHFGWI9Shdm6pbGxsWm9p/8jWiWVrbSBPS07ZDBK4g45\nvHDhwoVhb5SGi1KCrbV21FFHdbVopVpr8cGO8+fPD3ujtUbRGqZ77rknfP2WLVu6WrYyKjoE8vjj\njw97o3VJ0aGZrcUpv2gFU2vxv4HZZ/Pmm2+O9LNas8YJgHcpQwuAMgwtAMowtAAoY84HMTLRepaZ\ncsstt4T1hx9+eORrXHrppV3tfe973zu+Jxgq+5J9yDqfLOwUOfroo7tadkZWFArIAlRR8CN7b1GQ\nYu3atV0tO+cruq8oCNJaHLpYvHjxyNfduHFj2Bv9WWThl2jNVhbEiD6zIYGLjCctAMowtAAow9AC\noAxDC4AyDC0AypAenGUPPvhgV/vCF74Q9kYHsWUHxN1www1dLUv1wEyIVjC1Fv89jNYMtRYnzrLk\n3pDrRgdGZqukoutm69Puv//+rnbXXXd1teeffz58/erVq7vaaaedFvauWLGiq2UpzOh+s38PooMz\no/Rha8PSnTPFkxYAZRhaAJRhaAFQhqEFQBmCGLMs+uI2ClxkrrnmmrAenbUDe4PoS/1oHVAmW/0T\nhRCyMEh0D9mZUdFqpmeeeSbsvf3227vaunXrutrJJ58cvv6cc87pahdccEHYG62dyj7H6P1G54+1\nFgdSsrO3onp2LtlM8aQFQBmGFgBlGFoAlGFoAVCGoQVAGdKDM+Tqq68O67/85S9HvsZXvvKVrva1\nr33tHd8T7AlR+i9bzRSl4bLeKHWbJQ2j5F123cnJya52xx13hL1//vOfu1p0gOz73//+8PVRevDA\nAw8Me6MDLrP3ECUrs88mSgRmh2lGScEssRn9vOwespVaYe/InQCwhxlaAJRhaAFQhqEFQBmCGNNg\n69atXe0Pf/hD2Bt96Xn44YeHvddff31X2xvOs4Ehoi/fs/Oaot7sLKtIdp5WdGZUFmKI1jBFgYvW\nWlu/fn1Xu/DCC7vaypUrw9efcMIJXS1b6xadCXbIIYeEvQsXLuxq0Xqq1uLwy+LFi8PeIaGaKKCR\nnf81hCctAMowtAAow9ACoAxDC4AyDC0AypAenAZXXnllV3v11VdHfv2Xv/zlsL506dJ3fE+wtxiS\nHozqWeIsum7WGyXkouRfa63ddtttXS37fT7ooIO62hlnnNHVVqxYEb4+WtmUHewYJYezpOHGjRtH\n7o0SiFkKM0oEZmucolRhljTM1jtFPGkBUIahBUAZhhYAZRhaAJQhiDHAAw88ENbvvvvuka/x0Y9+\ntKtde+217/SWoKTsy/sholVF2Rf9ExMTXS07I2vt2rVdbcOGDWHvRRdd1NUuueSSrnb00UeHr4/u\nNzvLav78+V0tW7cUhS6yc7oWLVrU1bJAS7SyLvpzaC0O1QwJXGQ8aQFQhqEFQBmGFgBlGFoAlCGI\nkZicnOxq1113Xdg75LyfM888s6s5I4u5JvtCPtoGccABB4S9UQAgu2600SIKXLTW2nPPPdfVshDD\niSee2NWWL1/e1bLf8eiMqyhw0Vr8OWSBieizWbJkSdgbic7uaq21zZs3j3wP++zTPxPZiAHAnGJo\nAVCGoQVAGYYWAGUYWgCUIT2YuPHGG7vamjVrRn791VdfHdatbIL8PK3oHKfsbKcoaTg1NRX2PvPM\nM13tkUceCXujJNunP/3psDdayxadg5edZbVz586ulr3faN1SlsYbGxvragsWLAh7I6+88srIvdF7\naC1PQe4uT1oAlGFoAVCGoQVAGYYWAGUIYiSuv/763Xr9d7/73bBuZRNzTbTOJ1v9E4Ugsi/6o3DD\n888/H/bed999XS1a7dRaa6eeempXO+uss8Legw8+eKT7ys7jitZDZYGJaF1cFvCIAhrZPbz22mtd\n7dlnnw17oz+fVatWhb3Rn7HztACYUwwtAMowtAAow9ACoAxDC4AypAdnyNatW8N6lKSaDlHiKEto\nvfHGG10tSyFFogMyW2vthhtuGPkakex+oyTnTK2IYfpFSbbs9yD6u5UdAhml/x566KGw97HHHutq\n2UGHUUovSxr+7W9/62rbtm3ratkhktG6pWwVVfSZZUnD6L1l7yFa2RQdTtlaa+edd15Xyw7Bje43\nS09n66jC647cCQB7mKEFQBmGFgBlGFoAlCGIMUOOOuqoWf1511xzTVc78sgjw96XX365q/3gBz+Y\n9nuaLtFn+fnPf34P3AnvRPQle3aeVhSwyc6XWrJkSVc76KCDRr5u9uX/iy++2NXuuOOOsDdaMbV5\n8+auloVJojOyslBUtDIq+2wmJia6WhauiMIRK1asCHujcEUWaMne8+7ypAVAGYYWAGUYWgCUYWgB\nUIahBUAZ0oOJq666qqvddNNNe+BORnPjjTfOyHWjdFK2biny2c9+NqxH62AyF1xwwci91Bb93YrW\njrUWp+lOOOGEsPfss8/uai+88ELYG61huvvuu8PeTZs2dbUoUZil/KL0YLaaaeHChV1t2bJlYW90\njexzPProo7talh489NBDu1qWEoze85B1TRlPWgCUYWgBUIahBUAZhhYAZcx76623ZutnzdoPmik/\n+9nPwnp2nsyoHn744bC+u6uVvvrVr4b1448/fuRrfPjDH+5qhx122Du+p1mw+9/08v81MTEx8u9z\ntPonW+MUrRTKeqNgwq5du8Lep556qqtFZ2y11tp//vOfrhadOdVavPIpOksvC0FEoY3srLhjjjmm\nqx1++OFhb/SZZSuuon8PsjV00b1FgZjW4jDIkPMEly1bFv4ue9ICoAxDC4AyDC0AyjC0ACjD0AKg\nDOlB3m2kB2fBkPRgJEuRRam3rDdKp2VJw+igwmiFUvbzsn8no8MWo0McN2zYEL4+WhmVHd4apf+y\ngx2jVGK2bilanRUlMzPZiqrdXdkkPQhAeYYWAGUYWgCUYWgBUIbztIBZlwUmonrWGwUeogBCa/Gq\ntSxAMORMr+h8qbGxsa6WrZeK3kO2xim6r+jntxZ/DlmYZGpqKqxHomtMxxlZQ3jSAqAMQwuAMgwt\nAMowtAAow9ACoAzpQWCvEa1QypJ3Q1Y+RYm8LBEYpQqzRF9Uj5KKQ9YtZYcqRp9DlgiMerP3G8ne\n75BrzBRPWgCUYWgBUIahBUAZhhYAZQhiAHu1bN3S/vvv39Wy0EZkyPqh7LqjrkvKVlFFZ1xFq52y\nexhyHmIWUok+h70hcJHxpAVAGYYWAGUYWgCUYWgBUIahBUAZ0oNASVEiL0sE7m5SMEv/RaJEX5aA\nHHJf0WqlLNU42wczziZPWgCUYWgBUIahBUAZhhYAZcwbsgYEAPYkT1oAlGFoAVCGoQVAGYYWAGUY\nWgCUYWgBUIahBUAZhhYAZRhaAJRhaAFQhqEFQBmGFgBlGFoAlGFoAVCGoQVAGYYWAGUYWgCUYWgB\nUIahBUAZhhYAZRhaAJRhaAFQhqEFQBn/A+e+Bq9IX7diAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f88440c4c50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs, \"./my_model_cache_frozen.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Tying weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "It is common to tie the weights of the encoder and the decoder (`weights_decoder = tf.transpose(weights_encoder)`). Unfortunately this makes it impossible (or very tricky) to use the `tf.layers.dense()` function, so we need to build the Autoencoder manually:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 150  # codings\n",
    "n_hidden3 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01\n",
    "l2_reg = 0.0005\n",
    "\n",
    "activation = tf.nn.elu\n",
    "regularizer = tf.contrib.layers.l2_regularizer(l2_reg)\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "\n",
    "weights1_init = initializer([n_inputs, n_hidden1])\n",
    "weights2_init = initializer([n_hidden1, n_hidden2])\n",
    "\n",
    "weights1 = tf.Variable(weights1_init, dtype=tf.float32, name=\"weights1\")\n",
    "weights2 = tf.Variable(weights2_init, dtype=tf.float32, name=\"weights2\")\n",
    "weights3 = tf.transpose(weights2, name=\"weights3\")  # tied weights\n",
    "weights4 = tf.transpose(weights1, name=\"weights4\")  # tied weights\n",
    "\n",
    "biases1 = tf.Variable(tf.zeros(n_hidden1), name=\"biases1\")\n",
    "biases2 = tf.Variable(tf.zeros(n_hidden2), name=\"biases2\")\n",
    "biases3 = tf.Variable(tf.zeros(n_hidden3), name=\"biases3\")\n",
    "biases4 = tf.Variable(tf.zeros(n_outputs), name=\"biases4\")\n",
    "\n",
    "hidden1 = activation(tf.matmul(X, weights1) + biases1)\n",
    "hidden2 = activation(tf.matmul(hidden1, weights2) + biases2)\n",
    "hidden3 = activation(tf.matmul(hidden2, weights3) + biases3)\n",
    "outputs = tf.matmul(hidden3, weights4) + biases4\n",
    "\n",
    "mse = tf.reduce_mean(tf.square(outputs - X))\n",
    "reg_loss = regularizer(weights1) + regularizer(weights2)\n",
    "loss = mse + reg_loss\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train MSE: 0.016999\n",
      "1 Train MSE: 0.0166273\n",
      "2 Train MSE: 0.0163764\n",
      "3 Train MSE: 0.0172502\n",
      "4 Train MSE: 0.016203\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 5\n",
    "batch_size = 150\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = mnist.train.num_examples // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch})\n",
    "        mse_train = mse.eval(feed_dict={X: X_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_train)\n",
    "        saver.save(sess, \"./my_model_tying_weights.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa0AAAFxCAYAAADAqvdjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAG4hJREFUeJzt3UtslPX3x/FvubS0tLQUWqAIpYhcFFBUwGt+xBhjYqJR\n4wJZaNBE48JEEzWYmOhKd8rGqNEoXhYmJmrUqFHRBBQNXiAoXiqXclFsgWLvUMDf4r/65XyO/+dh\nOkNP+34tT87MPDOlPT6Zj+db9s8//yQAACIYc7YvAACArBhaAIAwGFoAgDAYWgCAMBhaAIAwGFoA\ngDAYWgCAMBhaAIAwGFoAgDDGlfC1WL2BUig72xcwGnR0dPD7jKJqaGiQv8vcaQEAwmBoAQDCYGgB\nAMIo5XdaAJCbdxJFWVlhX196z6vq3mtlvQavT71Wsd7vSMGdFgAgDIYWACAMhhYAIAyGFgAgDIYW\nACAM0oMAhrVipea85z19+nRBzzFmjL0XGIrXGjt2bOZedQ3ea506dSrz8yqlTjtypwUACIOhBQAI\ng6EFAAiDoQUACIMgBoBhzfuiX1EBhJR02GD8+PGZnzdPCCLr66eU0uDgoKl573fcOPvn2nu/AwMD\nppYnGOG9X/UcpV4vxZ0WACAMhhYAIAyGFgAgDIYWACAMhhYAIAzSgwCGjTwHMKpelbDzelVyL6WU\nKioqTO348eOyVyUQ1TV4abzy8nJZz6q3t1fWVVpRva+U9HqnkydPyt48iUtlKFY+cacFAAiDoQUA\nCIOhBQAIg6EFAAiDIAaAkvO+kFehAC/EoMICXoAgT2+eFUhZz8OqqqqSdfW83nWp4Ii3xkkFJrz3\n4IVXFPV+vc8gz8qnXKu6MncCAHCWMbQAAGEwtAAAYTC0AABhMLQAAGGQHgRQcnlSZN4BiqrurTXq\n7Ow0NZUSTEmvO6qtrZW9dXV1mR7vpeNOnDhhanlSfl4qUSUQvRRm1gRkSvnWOKnVV957Y40TAGBE\nYmgBAMJgaAEAwmBoAQDCGFVBjK+//trU1q9fL3tnzpxpapWVlbL3jjvuMLX6+nrZ69WB0STPGicv\nMNHR0WFqhw4dkr379u0ztf7+ftk7ceJEU6uurpa9KnTR2Nhoat7ZXT09PabmrWZqaGgwtTwBD/W+\nUkpp8uTJpjZlyhTZq34+3hoo1eu9N4IYAIARiaEFAAiDoQUACIOhBQAIg6EFAAijLM/hWwUq2Qt5\nFixYYGqtra1FeS1v7ctll11WlNcrljlz5pjaunXrZO/s2bOLfDWZZI8h4Yx1dHRk/n1Wf2O81UFq\n3ZJKCaaU0u7du03N+30+duyYqXmpRLUCyVsPpaiE3MGDB2Wver/Tp0+XvfPmzTM1LwGp6i0tLbL3\nP//5j6ldeOGFsnfChAmyrqg1W3nSgw0NDfJ3mTstAEAYDC0AQBgMLQBAGAwtAEAYo2qN0zvvvGNq\n27Ztk70XXHCBqf3000+y95tvvjG1d999V/Z+/PHHpuZ9Qbpnzx5Zz8pbrzJjxgxT279/f+bnVeGM\nlFJ65JFHMj8HRh4v1KXq6qyllPQX8t55WirEoNYXpaQDAGoFU0o6iNHX1yd71XtTNS8wod6vd+6V\nCpO0tbXJ3l9//dXUFi1aJHubmppMTYXWUtLnaXnXq+rqs/2351C40wIAhMHQAgCEwdACAITB0AIA\nhMHQAgCEMarSgyo94yVqlKVLl8r66tWrTe2pp56SvXv37jU1Lz2o1tTkUV5eLusqPehdg1qhs3Dh\nwoKuCyOTd5CfqqsUWko6XeYly9S/45qaGtmrkrTeNXR3d5ualx5Uz6HSjl5qTl2XdwBje3u7qalD\nJL26dxClSlF6P0v1s/B+PupzyJMS9HCnBQAIg6EFAAiDoQUACIOhBQAIY1QFMUrJO3cmT4ghT0gk\nD7V26vDhw7J35cqVpnbdddcN+TUhFrWqyPvyPk+vCg81NDTIXnVmXZ7zAb2gklr55AUpVBBDvTfv\nHKmqqipT88IKv/zyi6lt2bJF9tbV1ZnazJkzZa8KYXkhlTxBikLP0/JwpwUACIOhBQAIg6EFAAiD\noQUACIOhBQAIg/TgCNbb2yvrN998s6mdPn1a9j7zzDOmVllZWdiFITyV9vKSe+rflpcW8w5mVPKk\nElWa10sP5kn/qWtQqTnvtVR60FsZdeTIEVPz1jipZOX8+fNlrzoEUl1XSnrtlHegp/rM8qQEPdxp\nAQDCYGgBAMJgaAEAwmBoAQDCIIgxgr3yyiuyfujQIVPzzvBpbm4eykvCKKS+fFdhhZT0mU95whXe\n+jQVmPBWFangiLfGqb+/39RU6MJ7DwMDA6Z28OBB2bt161ZTU2dspZTS3LlzTc07D7C+vt7UvM9R\nvY884QovrMMaJwDAiMTQAgCEwdACAITB0AIAhMHQAgCEQXpwhNi1a5epPfjgg5kf7x0mN3369DO+\nJowuXgIsz8onb12SolY+qTVDKaV04sQJU/MSgSrR19XVJXtV0nDixImm5n02atXaRx99JHs3btxo\nap2dnbJ31apVprZ48WLZ661sUlTq0/vMVRKU9CAAYFRhaAEAwmBoAQDCYGgBAMIgiDFCvPfee6am\nvghNKaXbbrvN1NTaF6BYvC/e1Rf1eVYKeeEKFcTwVkmp0IXXW1NTY2rV1dWm5r3fP//809S8UNSO\nHTtMbeHChbJXrWyaNm2a7FWBFu+MLBW6GDt2rOz1/v4UijstAEAYDC0AQBgMLQBAGAwtAEAYBDGC\n8b7cfPvtt01NfcGaUkpPPvmkqXlfpgKFUuEKb/OF2jCR53n7+voyPz5P2EBtuUgp+1lU3d3d8vGt\nra2mps67SymlxsZGU7v66qtl70UXXWRqdXV1sjfPGVkq0JLnXLI8G0883GkBAMJgaAEAwmBoAQDC\nYGgBAMJgaAEAwiA9GMxLL70k65s2bTK122+/XfaysglnW57ztLxEoUr/eSnYPOdAqTScWtfkPYc6\nj+unn36Sj9+8ebOpeWd3rVixwtSuuOIK2dvS0mJqXspPfY7qPaSkP19vdZb6GXs/yzypQu60AABh\nMLQAAGEwtAAAYTC0AABhlHlfiBZByV5opNi2bZupLV++XPaqL4q//fZb2TvCgxh6/wyGVEdHR+bf\nZ/U3xvu7o9YHeb1qpVB5ebns9c7Dyspb46QCBL/99pupbdiwQT7+iy++MLX58+fL3uuvv97Urrnm\nGtk7Y8YMU/MCE2r1lbfiSr1f73nVOisvKKN+xo2NjfJ3mTstAEAYDC0AQBgMLQBAGAwtAEAYDC0A\nQBiscRoG+vv7ZX316tWm5qWg1qxZY2ojPCWIwLx/x2rNj0oJppTv4NI8a4LU83rX29HRYWoffPCB\nqb3zzjvy8ZWVlaZ26aWXyt5Vq1aZmjoYMiX9mXnvobe319TyrFvy1kPl4R06Ka+h4FcDAKBEGFoA\ngDAYWgCAMBhaAIAwCGKUmPqC84YbbpC9v/76q6ktWrRI9j7xxBOFXRhQJOrLey8UoNb5qHVA3nN4\nz6uuQYUgUtJBDC8M0traamrqbLv9+/fLx6s1TF4QY9q0aabmBRjUaqXu7m7Zqz5z7/2qn8Xg4KDs\nraiokPVCcacFAAiDoQUACIOhBQAIg6EFAAiDoQUACIP0YIkdPXrU1NRBcJ7XXntN1uvr68/0koCi\nUuk0b62SSsN5vWp9kJdkU4lA78BIdQ1HjhyRveqg1p07d5padXW1fPzFF19saosXL5a9KrnnrVtS\nvMMa1eebJ7HpHZBZrAOGudMCAITB0AIAhMHQAgCEwdACAIRBEKNI/v77b1m/7LLLMj/H66+/bmrL\nli0742sCzgb1hby34kcFKbzzmlTdO2NLhQ28YMLAwICp7dixQ/Z+/fXXpnbs2DFTW7JkiXz8tdde\na2pTpkyRvePG2T/X6lpT0oEJb22V+vl467DUzy3PuWbe2qk8oQ3utAAAYTC0AABhMLQAAGEwtAAA\nYTC0AABhkB4skpdfflnWd+/enfk5rrrqKlPz0jfAcKWSYV5aTCUCVWrOq+c5XNJL+O7Zs8fUVEow\nJZ0qVCuQli5dKh/f3Nxsat5no9KOXgJSPUeelJ8nz3N467eUPH/XuNMCAITB0AIAhMHQAgCEwdAC\nAIRBEGMItLa2mtrjjz9e+gsBhiH1JXuetT3emVFdXV2Zn6O7u9vU+vr6ZO+PP/5oavv27ZO96iwp\ntbJpxYoV8vEqeOKtZlLBk1zrj5xghApzeOEXFcTwfj7q2oYiSMadFgAgDIYWACAMhhYAIAyGFgAg\nDIYWACAM0oNDYNOmTaaWJ9m0aNEiWfcObQMiKTQx5q0qUrw1ToqX0isvLze1+vp62bty5UpTa2pq\nMrWWlhb5eHW93nvI8zmeOHHC1LzDNNXz5lqrlGNd01DgTgsAEAZDCwAQBkMLABAGQwsAEAZBjBK7\n4oorTO2TTz6RvQQxAJ9aKeSd96RWClVXV8veBQsWmFpjY6PsVYGrWbNmmdrUqVPl41XgwQtxVVRU\nyLqSZ+WTt7JpuOJOCwAQBkMLABAGQwsAEAZDCwAQBkMLABBGWZ5DxApUshfCqFb4KXP4f3V0dJTu\nD4fzN0odPuitH1J17/BCtUbp+PHjslcl+lSC0VvNpFYgee9X9eZ5v56hOJixGBoaGuSFcacFAAiD\noQUACIOhBQAIg6EFAAijlEEMAAAKwp0WACAMhhYAIAyGFgAgDIYWACAMhhYAIAyGFgAgDIYWACAM\nhhYAIAyGFgAgDIYWACAMhhYAIAyGFgAgDIYWACAMhhYAIAyGFgAgDIYWACAMhhYAIAyGFgAgDIYW\nACAMhhYAIAyGFgAgDIYWACAMhhYAIAyGFgAgDIYWACAMhhYAIAyGFgAgDIYWACAMhhYAIAyGFgAg\njHElfK1/SvhaGL3KzvYFjAZtbW38PqOompub5e8yd1oAgDAYWgCAMBhaAIAwSvmdFgAMe//8Y7+u\nKysr7KtS7/HqtVQt7/OOZNxpAQDCYGgBAMJgaAEAwmBoAQDCYGgBAMIgPQhgVBozRv83++nTpzPV\nUkpp3Dj7J1Sl/06dOiUfnycpqK7Be3xlZWXma1B177NRSp1g5E4LABAGQwsAEAZDCwAQBkMLABAG\nQYwh8MYbb5hab2+v7P3uu+9M7YUXXsj8Wo899pisX3PNNaa2atWqzM8LjAReMGFwcNDUTp48KXt7\nenpMzQtiHD9+3NTGjx9val4IQvUODAzIXhV4qKioyNw7YcIE2atCG14QYzisjeJOCwAQBkMLABAG\nQwsAEAZDCwAQBkMLABBGWZ41IgUq2QsVy3333Sfrzz//fImvxDr//PNNbfPmzbK3tra22JdzNp39\neNMo0NbWNix/n1WaL6WUjhw5Ymq///677G1razO1P/74Q/Z2d3ebmvqb6iUVVfqvqalJ9i5evNjU\nZs2aJXtrampMzUsPqlVUKtWYUmnTg83NzfLFuNMCAITB0AIAhMHQAgCEwdACAITBGieHCl0MReBi\n2bJlpnbrrbfK3tbWVlPbsGGD7N25c6epvfXWW7L3rrvu+rdLBIYVb4VSX1+fqbW3t8veH3/80dR+\n/vln2bt3715T6+rqkr1qPZRageSFIFSwwXu/U6ZMMbW5c+fK3vr6elPzVj719/ebWp51WF44Q9XH\njh2budfDnRYAIAyGFgAgDIYWACAMhhYAIAyGFgAgjFGfHty3b5+sv/jii5mfY/ny5ab20Ucfyd6q\nqipTKy8vl73q4Dhv9cyXX35paocPH5a9QCQqsZaSTtmptUop6VSh19vQ0GBqK1eulL3nnXeeqanf\n2927d8vHq1Sjt4oqa1IxJZ0UVH97UkrpxIkTpqaSmd415DmI0ksJeqlChTstAEAYDC0AQBgMLQBA\nGAwtAEAYoz6I4YUV1BoTFbhIKaVPP/3U1Kqrqwu7sJTSK6+8Ympbt27N/Pibbrqp4GsACpXnzD51\n7pQXNlCBBe/cqmnTppnajBkzZO/8+fNNTZ1Xl5IOUam/Kd56KRWC6OzslL1qlVRvb6/sVUEKLwSh\nnsMLg/T09JhaZWWl7PXqCkEMAMCIxNACAITB0AIAhMHQAgCEwdACAIQx6tODF198sayrBJC3bilP\nSiYPtUpKpY2A4SzPAX8qaeilD8ePH29qkyZNkr11dXWm5iV8VXpQPT4lnab766+/TE2ta0pJH0Sp\nDnBMSV+vl6xU65bUYY8p6c/XSw+qwyyH4hDIPLjTAgCEwdACAITB0AIAhMHQAgCEMeqDGJ7a2tqS\nvdZrr70m69u3b8/8HNddd52pnXvuuWd8TcBQyRPEUF/Ue2EDtbKppqYm8zXMnDlT9qrAw8DAgOxt\nbW01tffffz9TX0o6dLFkyRLZe+GFF5ra7NmzZa/6zLz3oIIYKuSSkg54eEE0FVzL82/Bw50WACAM\nhhYAIAyGFgAgDIYWACAMhhYAIAzSgyX2ww8/mNo999wje9UqFe/guvXr15ualwACSkml07wU2alT\np0zNW/1z+vRpU6uoqJC9VVVVpuatcVLX4B3iuHHjRlPbtm2bqXnJvQULFpiat1pOrZfy1lap9VLe\nAZkq5Zdn3ZLXm+c58hwUyp0WACAMhhYAIAyGFgAgDIYWACAMghgltmXLFlPzzq5R7r33XllXX9IC\nw4EKTHirmVRgwjtDToUu1HlP3vOq60oppd7eXlPzzsPasWOHqakQRGNjo3z8vHnzTG3ZsmWyV4Uu\nvL8df//9t6l54RcvvKKoz8wLUeQJ4ORZ78SdFgAgDIYWACAMhhYAIAyGFgAgDIYWACAM0oNFsnbt\nWll/8803Mz/HAw88YGoPP/zwGV8TMFTyJMO8pKCiVg15z6vWD3nUNXjX1dbWZmqbNm2SvX/99Vem\n521qapKPb2lpMTVvNVNfX5+pdXV1yV6VuPQOyJw4caKpeSufuru7TU2tvUpJJzk5BBIAMKowtAAA\nYTC0AABhMLQAAGEQxBgCam3Lhx9+KHvVuTrTpk2TvY8++qip5fnyGSiWQr9QzxPO8NYtqfVD3koi\nFRY4duyY7P3+++9N7cCBA7J33Dj7J1SFLqZPny4fP3nyZFlXOjs7Tc1b41RbW2tq3vlh6mfpfeYq\ngOP1quf1Vj6xxgkAMCIxtAAAYTC0AABhMLQAAGEwtAAAYZAeHAK33XabqbW3t2d+/P333y/r9fX1\nZ3xNwHDmJc5UqnDs2LGyVx3s6PWqQxF37dole3/44QdTU8m9lPT7UElBddhjSnqN0/jx42Wveg/e\n56g+B+95VaLPW+Ok1kN5n7n6WXrXmwd3WgCAMBhaAIAwGFoAgDAYWgCAMAhi5PDdd9/J+hdffJH5\nOW655RZTe/DBB8/0koARRX2pr85l8ure2U5//vmnqX311Vey9/PPPzc1b+3UJZdcYmqXXnpppr6U\nUpoxY4apeWdkqRVuXghCfTbeCiV1Ttfhw4dlr3o9tcoqJR26YI0TAGBUYWgBAMJgaAEAwmBoAQDC\nIIjh6O/vN7V169bJXvV/iXvUF7KckQX8HxWkqKyslL3qy3v1e5tSSh0dHaa2fft22XvkyBFTmzNn\njuydO3euqc2fP9/Upk6dKh+vwgre3xO1AcT7bFQ4wnte9X7V55VSSpMmTTI1b9OG2qrhhTby4E4L\nABAGQwsAEAZDCwAQBkMLABAGQwsAEAbpQcdzzz1nap999lnmx69du1bWWdkE+FS6zFvNpNJp3voh\nlRTcu3ev7FUrkBYsWCB7ly5damrnnHOOqXlrilRyT52blVJK1dXVpuadT6U+M7XKKiX9OfT09Mhe\nlYz0HD9+3NS8tVOscQIAjEgMLQBAGAwtAEAYDC0AQBgEMRyPPvpoQY9/+umnZZ2VTYAfrlB1b/2Q\nCiy0tbXJ3p9//tnUvHOr1Mql5uZm2Ttz5kxTU+GIP/74Qz7+wIEDpuadOaV4n01nZ6epbd26Vfaq\n0MX06dNlb29vr6lNnDhR9qqVT0OBOy0AQBgMLQBAGAwtAEAYDC0AQBgMLQBAGKQHi8RbgzJmTHH+\nO6GiosLUvJUpKqGlVq54vIP21q9fn/k5FO96VZLTO3gOw49Kw3mpN/Vz7evrk70qIbdr1y7Zq1KF\n3rqkyZMnm5r3b37fvn2mptYieSuU1PN6vwdHjx41te7ubtm7Z88eU2tvb5e9iloZ5dW9gyjV3zov\nGckaJwDAiMTQAgCEwdACAITB0AIAhEEQo0jUepdiuvfee02tqalJ9h46dMjUnn322SG/pqGiPsu7\n7777LFwJzoY8ZzCpM7ZSyrceSoUmNm/eLHt3795taiqE5QWwqqqqTM0LQagQg/pdTkmHNryww/Ll\ny02tvr5e9qo1dIODg7JXnUs2FEE07rQAAGEwtAAAYTC0AABhMLQAAGEwtAAAYZAedKxZs8bUXn75\n5bNwJdk899xzRXnecePsPxEvzaXceeedsn755Zdnfo4rr7wycy+GH5Va85Jsqq5SaCmlVFtba2qz\nZs2SvStXrvy3S/wf6mDG1tZW2btlyxZTU4dAqpRgSvqwxdmzZ8tetfJJ/X6mpFdRTZkyRfbOnTs3\n8zUo3hqnPKuZ8uBOCwAQBkMLABAGQwsAEAZDCwAQRpl3vkkRlOyFiuXVV1+VdW8dTFbbt2+X9UJX\nKz300EOyPm/evMzPceONN5paY2PjGV9TCRTn21/8j7a2toJ+n711PirEMGnSJNmr/nYdO3ZM9qrV\nTL/88ovsVWdv7dy5U/Zu27bN1NR784IYCxcuNDUvMKHOvKurq5O9KojhhVRaWlpMTa1r8l7PO9tO\nBTG8n7v6Wc6ZM0f+LnOnBQAIg6EFAAiDoQUACIOhBQAIg6EFAAiD9CBGGtKDJZAnPaj+xnirwFTd\nS7Kp9UHqsMeUdMLNS/329vaaWldXl+w9ePCgqbW3t5ua93dWpQq99Ufqc/DSg+qz8T5HdW1eyk99\njl56sFDNzc2kBwEAsTG0AABhMLQAAGEwtAAAYXCeFoCiUsECta7Jc/LkSVlX4YiKiorMz1tTUyPr\nKhzR0NAge9W5UyoM4gVEVAiip6dH9qrPzAu0qOf1PkfV64VBVEDDC5lwnhYAYNRjaAEAwmBoAQDC\nYGgBAMJgaAEAwiA9CGDYUCk7L/WmknNer1ph5KX01FqiwcFB2auuV13XUKQa1dop7/1mva6UdCox\nT/KvWClBD3daAIAwGFoAgDAYWgCAMBhaAIAwCGIAGDbUl/reF/151kOpM7K887QUdT5VSnqFkboG\n77q8c6sUFa7wVijlOSex1EGKQnGnBQAIg6EFAAiDoQUACIOhBQAIg6EFAAiD9CCAYc1Lwqk0nZeE\nUyuM1Gonr9eTNXnnpQfzJAILef2RhDstAEAYDC0AQBgMLQBAGAwtAEAYZXm+9AMA4GziTgsAEAZD\nCwAQBkMLABAGQwsAEAZDCwAQBkMLABAGQwsAEAZDCwAQBkMLABAGQwsAEAZDCwAQBkMLABAGQwsA\nEAZDCwAQBkMLABAGQwsAEAZDCwAQBkMLABAGQwsAEAZDCwAQBkMLABAGQwsAEAZDCwAQxn8B9uYU\ngGqq1QIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f88454790f0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs, \"./my_model_tying_weights.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Unsupervised pretraining"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 150\n",
    "n_outputs = 10\n",
    "\n",
    "learning_rate = 0.01\n",
    "l2_reg = 0.0005\n",
    "\n",
    "activation = tf.nn.elu\n",
    "regularizer = tf.contrib.layers.l2_regularizer(l2_reg)\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "y = tf.placeholder(tf.int32, shape=[None])\n",
    "\n",
    "weights1_init = initializer([n_inputs, n_hidden1])\n",
    "weights2_init = initializer([n_hidden1, n_hidden2])\n",
    "weights3_init = initializer([n_hidden2, n_hidden3])\n",
    "\n",
    "weights1 = tf.Variable(weights1_init, dtype=tf.float32, name=\"weights1\")\n",
    "weights2 = tf.Variable(weights2_init, dtype=tf.float32, name=\"weights2\")\n",
    "weights3 = tf.Variable(weights3_init, dtype=tf.float32, name=\"weights3\")\n",
    "\n",
    "biases1 = tf.Variable(tf.zeros(n_hidden1), name=\"biases1\")\n",
    "biases2 = tf.Variable(tf.zeros(n_hidden2), name=\"biases2\")\n",
    "biases3 = tf.Variable(tf.zeros(n_hidden3), name=\"biases3\")\n",
    "\n",
    "hidden1 = activation(tf.matmul(X, weights1) + biases1)\n",
    "hidden2 = activation(tf.matmul(hidden1, weights2) + biases2)\n",
    "logits = tf.matmul(hidden2, weights3) + biases3\n",
    "\n",
    "cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "reg_loss = regularizer(weights1) + regularizer(weights2) + regularizer(weights3)\n",
    "loss = cross_entropy + reg_loss\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "correct = tf.nn.in_top_k(logits, y, 1)\n",
    "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "pretrain_saver = tf.train.Saver([weights1, weights2, biases1, biases2])\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Regular training (without pretraining):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train accuracy: 0.946667 Test accuracy: 0.9344\n",
      "1 Train accuracy: 0.973333 Test accuracy: 0.9274\n",
      "2 Train accuracy: 0.953333 Test accuracy: 0.9421\n",
      "3 Train accuracy: 0.98 Test accuracy: 0.9435\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 4\n",
    "batch_size = 150\n",
    "n_labeled_instances = 20000\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = n_labeled_instances // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            indices = rnd.permutation(n_labeled_instances)[:batch_size]\n",
    "            X_batch, y_batch = mnist.train.images[indices], mnist.train.labels[indices]\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"Train accuracy:\", accuracy_val, end=\" \")\n",
    "        saver.save(sess, \"./my_model_supervised.ckpt\")\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})\n",
    "        print(\"Test accuracy:\", accuracy_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Now reusing the first two layers of the autoencoder we pretrained:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train accuracy: 0.913333\tTest accuracy: 0.9105\n",
      "1 Train accuracy: 0.986667\tTest accuracy: 0.9366\n",
      "2 Train accuracy: 0.946667\tTest accuracy: 0.9386\n",
      "3 Train accuracy: 0.98\tTest accuracy: 0.9503\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 4\n",
    "batch_size = 150\n",
    "n_labeled_instances = 20000\n",
    "\n",
    "#training_op = optimizer.minimize(loss, var_list=[weights3, biases3])  # Freeze layers 1 and 2 (optional)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    pretrain_saver.restore(sess, \"./my_model_cache_frozen.ckpt\")\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = n_labeled_instances // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            indices = rnd.permutation(n_labeled_instances)[:batch_size]\n",
    "            X_batch, y_batch = mnist.train.images[indices], mnist.train.labels[indices]\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"Train accuracy:\", accuracy_val, end=\"\\t\")\n",
    "        saver.save(sess, \"./my_model_supervised_pretrained.ckpt\")\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})\n",
    "        print(\"Test accuracy:\", accuracy_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Stacked denoising Autoencoder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: the book uses `tf.contrib.layers.dropout()` rather than `tf.layers.dropout()` (which did not exist when this chapter was written). It is now preferable to use `tf.layers.dropout()`, because anything in the contrib module may change or be deleted without notice. The `tf.layers.dropout()` function is almost identical to the `tf.contrib.layers.dropout()` function, except for a few minor differences. Most importantly:\n",
    "* you must specify the dropout rate (`rate`) rather than the keep probability (`keep_prob`), where `rate` is simply equal to `1 - keep_prob`,\n",
    "* the `is_training` parameter is renamed to `training`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 150  # codings\n",
    "n_hidden3 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01\n",
    "l2_reg = 0.00001\n",
    "dropout_rate = 0.3\n",
    "\n",
    "activation = tf.nn.elu\n",
    "regularizer = tf.contrib.layers.l2_regularizer(l2_reg)\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "is_training = tf.placeholder_with_default(False, shape=(), name='is_training')\n",
    "\n",
    "X_drop = tf.layers.dropout(X, dropout_rate, training=is_training)\n",
    "\n",
    "weights1_init = initializer([n_inputs, n_hidden1])\n",
    "weights2_init = initializer([n_hidden1, n_hidden2])\n",
    "\n",
    "weights1 = tf.Variable(weights1_init, dtype=tf.float32, name=\"weights1\")\n",
    "weights2 = tf.Variable(weights2_init, dtype=tf.float32, name=\"weights2\")\n",
    "weights3 = tf.transpose(weights2, name=\"weights3\")  # tied weights\n",
    "weights4 = tf.transpose(weights1, name=\"weights4\")  # tied weights\n",
    "\n",
    "biases1 = tf.Variable(tf.zeros(n_hidden1), name=\"biases1\")\n",
    "biases2 = tf.Variable(tf.zeros(n_hidden2), name=\"biases2\")\n",
    "biases3 = tf.Variable(tf.zeros(n_hidden3), name=\"biases3\")\n",
    "biases4 = tf.Variable(tf.zeros(n_outputs), name=\"biases4\")\n",
    "\n",
    "hidden1 = activation(tf.matmul(X_drop, weights1) + biases1)\n",
    "hidden2 = activation(tf.matmul(hidden1, weights2) + biases2)\n",
    "hidden3 = activation(tf.matmul(hidden2, weights3) + biases3)\n",
    "outputs = tf.matmul(hidden3, weights4) + biases4\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "mse = tf.reduce_mean(tf.square(outputs - X))\n",
    "reg_loss = regularizer(weights1) + regularizer(weights2)\n",
    "loss = mse + reg_loss\n",
    "training_op = optimizer.minimize(loss)\n",
    "    \n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train MSE: 0.019915\n",
      "1 Train MSE: 0.0135197\n",
      "2 Train MSE: 0.011092\n",
      "3 Train MSE: 0.0103048\n",
      "4 Train MSE: 0.00956749\n",
      "5 Train MSE: 0.00912805\n",
      "6 Train MSE: 0.00907707\n",
      "7 Train MSE: 0.00908674\n",
      "8 Train MSE: 0.00848128\n",
      "9 Train MSE: 0.00875844\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 10\n",
    "batch_size = 150\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = mnist.train.num_examples // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, is_training: True})\n",
    "        mse_train = mse.eval(feed_dict={X: X_batch, is_training: False})\n",
    "        print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_train)\n",
    "        saver.save(sess, \"./my_model_stacked_denoising.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa0AAAFxCAYAAADAqvdjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAG1hJREFUeJzt3Vts1WXWx/EHOfQEtBYtBQQUqoACGXSIoDKjRvGUSNSY\neLjQqInGRBNNZiaaGHWSiZfKjVGj8YCTGScaPAbPUTzgISjEaFXkICjUFmppy6kUfa/fd/3WvP+H\n7r3par+fy5W1//vfXXaXO/vnekb8/vvvCQCACI460jcAAEBRDC0AQBgMLQBAGAwtAEAYDC0AQBgM\nLQBAGAwtAEAYDC0AQBgMLQBAGKMq+Fys3kAljDjSNzActLW18X5GWTU3N8v3Mp+0AABhMLQAAGEw\ntAAAYVTyOy0AGFKKnpIxYoT+qlXVc07e8K6rDJUTPfikBQAIg6EFAAiDoQUACIOhBQAIg6EFAAiD\n9CCAYclL3g00vTdy5MjCjz9w4EDh3lGj7J/r3377rfB9eXJ61fPlPL4U+KQFAAiDoQUACIOhBQAI\ng6EFAAiDIAaAIS8nXKF6Dx06JHuPOsr+d78KTHjPrx7vPdfBgwdNbcyYMQO+rgqDeNdVoQv1XCn5\nIZGB4pMWACAMhhYAIAyGFgAgDIYWACAMhhYAIAzSgwAqLmf1j5dOUwk377o5STaV0hs9erTsVck7\nVfPSg2rlk0ofppRSX19f4euqpKC3Mkr15qQSWeMEAICDoQUACIOhBQAIg6EFAAiDIAaAQUN9qe+F\nIFQowAsmqBBDTmhDhTNS0kEKVevv75ePV/Wc0IgXUlH34PXmnAmmXgfvukWfK6W8NVt80gIAhMHQ\nAgCEwdACAITB0AIAhMHQAgCEQXoQQFmpxJiXIlMrjLzk3e7du02to6ND9qr6pEmTZG9DQ4OpjRs3\nTvbW1taamlqB5K1QUr3eGid1De+1UWk8LxGoer0Eo0phqlpKeenOHHzSAgCEwdACAITB0AIAhMHQ\nAgCEMayCGJ988ompLV++XPZOmTLF1GpqamTvddddZ2qNjY2y16sDQ1XOF/2q3t7eLns/+ugjU/vi\niy8KX3fy5Mmy9+ijjza1mTNnFu7NOXNKhSNyzu7ygg0qzDF+/HjZq36GUoRBVMjEO6eLNU4AgCGJ\noQUACIOhBQAIg6EFAAiDoQUACGNEKdZqFFSxJ/LMmjXL1DZs2FCW56qvr5f1RYsWleX5yuX44483\ntbvuukv2Tps2rcx3U4iOaaGk2trazPvZS8ipNJ13qKJKp3355Zey99133zW1tWvXyl6VQFQrmLy6\nStilpH+OXbt2mVpXV5d8vEoPegljlf7zeo899lhTmzhxouw966yzTK2lpUX2Kvv375f1sWPHmlp1\ndbXsVa9jc3Oz/AfFJy0AQBgMLQBAGAwtAEAYDC0AQBjDao3Tiy++aGrr1q2Tvaeccoqpff3117L3\n008/NbWXXnpJ9r7xxhumdsIJJ8jezZs3y3pR3ioWdY7Qtm3bCl9XhTNSSulvf/tb4WsgtpwAl1rz\no8IZKel1S8ccc4zsnTt37oCu662S6unpMTUvbNDZ2Vnout59qdexu7tb9k6YMMHUVOgjpZQ2btxo\nal4QQ4UuvPe4Cld4VMjEe81z8EkLABAGQwsAEAZDCwAQBkMLABAGQwsAEMawWuNUSV7aaMuWLabm\npQc3bdo0oHvwDlxT6UHvHjo6Okxt5cqVsnfZsmUZd1c2rHGqgB07dpj3s/e35NChQ6bmHaiqkobe\ne0mtfFK1lPSaoN27d8vePXv2mJqXxPXWUf1ffX19sq5ShTkJyFWrVsneV1991dS8FOa9995raqef\nfrrsraurMzXvNVA/h/dvRP1srHECAITH0AIAhMHQAgCEwdACAIQxrNY4VZJ3bszs2bMLX2POnDml\nup3/Ra2d2rlzp+xVX8guXbq05PeEWNTZWd55WooKXKSUUlVVlampdUAp6VCAenxKOqDh3a+6N2/9\nkAoW5IQV1D14QYxffvnF1FTgIiUdXpkxY4bsnTJlSuF7UD9vTriiFME/PmkBAMJgaAEAwmBoAQDC\nYGgBAMJgaAEAwiA9OISpdTQppXTZZZeZmpeOeuihh0zNW8GD4UOl3rxkmJdEU1Ryz0sPqt69e/fK\n3py0o6oXXdeUkr92SlFpR+/12rp1q6l9//33snf8+PGmtmjRItnb1NRkat5rk5P+U+u7cv4tePik\nBQAIg6EFAAiDoQUACIOhBQAIgyDGEPbUU0/Jeltbm6lNmDBB9k6fPr2Ut4QhYqDreLzgj6qrL/Q9\nXq+q19bWyl4VpMgJbeT0jR492tS8lWpvvvmmqbW2tsreJUuWmNqf//xn2atCG97rqMIvAz1rLBef\ntAAAYTC0AABhMLQAAGEwtAAAYTC0AABhkB4cIjZu3Ghqd955Z+HHr1mzRtabm5sP+56AXDkHB6q6\ndwjkvn37TM1LyKlEn3cPRddZeYfCqsMp1SGtKaX0/vvvm5r3/jz//PNNberUqbJXvWY9PT2yd8yY\nMabmJSNzVjblpFH5pAUACIOhBQAIg6EFAAiDoQUACIMgxhDxyiuvmJq3RuXKK680tRkzZpT8ngCP\nF4JQX957X+irc7a8s7dUgMA792rcuHGm5q2dUtdVoQLvvnbs2GFqKnCRUkqdnZ2mdsEFF8jexYsX\nm5r6uVLS9+v9vDkrtdQ1OE8LADCsMLQAAGEwtAAAYTC0AABhEMQIxgtXrFy50tS87QAPPPCAqXlf\nFAPlUPQcqv/Wq77U9zYrqH/f9fX1slcFCLz3h+pV2y+8DROfffaZqX3wwQeyV517tXDhQtk7adIk\nWVfUthDv51UbPLxtH+r35gU8cv498EkLABAGQwsAEAZDCwAQBkMLABAGQwsAEAbpwWCeeOIJWVeJ\no2uuuUb2srIJg5U6y8pLnOWsCRo1yv6p6+vrK3xdj0roqlVHmzZtko9ftWqVqXV0dMjeZcuWmdq8\nefNkr3odveRx0TPBUkqppqamcK+q56QEPXzSAgCEwdACAITB0AIAhMHQAgCEQRBjEFu3bp2p3Xbb\nbbK3oaHB1P7+97+X/J6AXCoc4X0hn/PlvToPy1tdpsIRXhBD8dYaqXp7e7up/fvf/5aPf+2110zN\nC1ecc845pjZ16lTZq14zL6Si6iq4kpIOqXgBj3KthuOTFgAgDIYWACAMhhYAIAyGFgAgDIYWACAM\n0oODgDqELaWUrr76alNTKaiUUrr22mtNjXVNGKy8RKBaP6QOHvR6vZVCOak3RR3AmJJOyK1evdrU\nXnjhBfl4db9/+tOfZO8f//hHU6urq5O96m+K9zoq3sGO6u+P95rnrIfKwSctAEAYDC0AQBgMLQBA\nGAwtAEAYBDEqTH0hfMkll8je7777ztTmzJkje++///6B3RhQQWPGjCnc661mUl/qe0ElxVtVpO7N\nu4dffvnF1N5++21T27Ztm3y8Cl1cfPHFsnfixImm5oVJVOjCe23Uz+utuFKvufc6liJ0ofBJCwAQ\nBkMLABAGQwsAEAZDCwAQBkMLABAG6cEK6+zsNLX33nuv8ONXrFgh642NjYd7S0DFlStZptK5KaXU\n399f+B5UGm7Pnj2y99VXXzW1d99919Samprk46+66ipT8w6BVPfrpfzUiivvEEh1mKaX7lSvjXfY\no0or5hz+6eGTFgAgDIYWACAMhhYAIAyGFgAgDIIYZbJ7925ZX7RoUeFrPPvss6a2YMGCw74n4EhQ\nX7J7X8jnUNfwrqsCBLW1tbJXrUBav3697H3ppZdMraenx9S81Uxnn322qXlrkVToImcdlhcmUQEN\nLxiRE2jhPC0AwLDH0AIAhMHQAgCEwdACAITB0AIAhEF6sEyefPJJWd+0aVPha5x11lmmVorUFTBY\n5Rzi6K1sUtRao5qaGtmr1hqtW7dO9n799demplY2LVmyRD6+vr7e1FR6MaWUent7Ta26ulr2qkSg\n93qpurfySaX/vARjzu8nB5+0AABhMLQAAGEwtAAAYTC0AABhEMQogQ0bNpjafffdV/kbAQYh9eW9\nWgeUUt7qH7XWyFvNdPDgQVPbu3ev7N28ebOpqfd4Sil1d3eb2vz5801t4cKF8vGKClyklBeYqKqq\nKvx86rXxQhQ5AY9y4ZMWACAMhhYAIAyGFgAgDIYWACAMhhYAIAzSgyXwwQcfmJpKFXnmzJkj696a\nGSA6bx1ZziGBI0eONDWVKPSer6OjQ/a+/PLLprZmzRrZq9ZOzZs3z9Sam5vl4xsaGkzNSw8q3hon\nlejLWQHnHUQ5GPBJCwAQBkMLABAGQwsAEAZDCwAQxuD9tm2IOuOMM0ztrbfekr0EMTDc5Kxxylkp\npK7rBRNOOukkU/PWMF144YWmtnTpUlPzgg0qsOWtuFI/r3f2luKdVeatghqsYt0tAGBYY2gBAMJg\naAEAwmBoAQDCYGgBAMIYkbM2ZYAq9kQY1orvqsFha2trG7LvZ5XI89ZDqTVKar2Uqnly0nze329V\nr+Df+pJobm6W72U+aQEAwmBoAQDCYGgBAMJgaAEAwqhkEAMAgAHhkxYAIAyGFgAgDIYWACAMhhYA\nIAyGFgAgDIYWACAMhhYAIAyGFgAgDIYWACAMhhYAIAyGFgAgDIYWACAMhhYAIAyGFgAgDIYWACAM\nhhYAIAyGFgAgDIYWACAMhhYAIAyGFgAgDIYWACAMhhYAIAyGFgAgDIYWACAMhhYAIAyGFgAgDIYW\nACAMhhYAIAyGFgAgDIYWACCMURV8rt8r+FwYvkYc6RsYDjo7O3k/o6waGxvle5lPWgCAMBhaAIAw\nGFoAgDAq+Z0WAAx6v/32W+Heo44q/X/3e8+vnst7/hEj7NdBv/+uv4bM+XkHAz5pAQDCYGgBAMJg\naAEAwmBoAQDCYGgBAMIgPQhgyPAScqrupeb6+vpM7dChQ7JXXaO6utrURo3Sf2pVr6p599Df3y97\nVapQJQq9a+T8vJXGJy0AQBgMLQBAGAwtAEAYDC0AQBgEMUrgn//8p6nt2bNH9q5du9bUHnvsscLP\ndc8998j6ueeea2pnn3124esCQ4EXxOjt7TW1rq4u2btlyxZTU+/blFI6cOCAqakQw8GDB+XjGxoa\nTE29l1NK6eSTTza1sWPHyl51DznBE6935MiRplaOVVb/DZ+0AABhMLQAAGEwtAAAYTC0AABhMLQA\nAGGQHsxw6623yvqjjz46oOvmpG/+8Y9/yPrKlStN7cMPP5S99fX1hZ8PONJy3h/79++X9ba2NlN7\n++23Ze9zzz1naq2trbK3qanJ1NT9emnixsZGU/N+BnXdefPmyd6amhpT81YzqUSgShR69+Bd11sb\nNVB80gIAhMHQAgCEwdACAITB0AIAhEEQw6FCFwMNXKSU0oIFC0ztiiuukL0bNmwwtaefflr2fvPN\nN6b2/PPPy94bb7zxv90iUHbel/Sq7p1FpQIL3d3dsleFLlasWCF7f/jhB1NraWmRvWeccYapqXBF\ne3u7fPyaNWtM7c0335S9M2bMMLUpU6bI3vHjx5va6NGjZa96fauqqmSvt45KUQENFfrIxSctAEAY\nDC0AQBgMLQBAGAwtAEAYDC0AQBjDPj24detWWX/88ccLX2PhwoWm9vrrr8ve2tpaUxszZozsVekb\nlWxKKaWPPvrI1Hbu3Cl7gSMtZ8WPt8apv7/f1H766SfZ671vlMsuu8zULrroItmrDlpV72fv78GP\nP/5oatu3b5e927ZtMzUvzbd3715T8xKBauWTlzRUvzcvEahSid4hnd6hkwqftAAAYTC0AABhMLQA\nAGEwtAAAYQz7IIYXVlBfGKrARUp6RczYsWMHdmMppaeeesrUPv/888KPX7Zs2YDvASgHL4ihvrz3\nghgHDhwwNS9AoM69uvDCC2WvqnvvffU+7+npMTXvzKnvv//e1Ly1VQ0NDaZWV1cne1XdO6dr3759\npua95qruhStUPSdw4eGTFgAgDIYWACAMhhYAIAyGFgAgDIYWACCMYZ8ePPXUU2VdpQq9dUtqDUop\nqFVSfX19ZXkuoFzUmh/v37FKFap1Td41vJVCf/jDH0zttNNOk73Nzc2mVl1dXfgefv75Z1PbtGmT\nfLz62dS9ppTS/PnzTa2+vl72quSel2BU9+D9rctZv6V+F14y0vsdK3zSAgCEwdACAITB0AIAhMHQ\nAgCEMeyDGB7vC85yWLFihayvX7++8DWWLl1qajNnzjzsewJyeSt6VADAC2KoAIG3Jkjx1qfNnTvX\n1CZPnix7VYBArTpKKaXe3l5TU2f0qfPuUtKrlaZPny57J06caGrea6PO2fJCFF54RckJyqh789ZD\n5fyO+aQFAAiDoQUACIOhBQAIg6EFAAiDoQUACIP0YIV9+eWXpnbzzTfLXnXI3aRJk2Tv8uXLTc07\nEA8oh5wEmLfOR6XTvNSbWp/mpdPGjRtnal5qTq0w6u7ulr0qKfjee++ZWmtrq3z87NmzTa2lpUX2\nqvezt5opJ7mnVlSp9KF3XY/3O1Zy1kPxSQsAEAZDCwAQBkMLABAGQwsAEAZBjApbs2aNqanAheeW\nW26R9ZNOOumw7wkohZx1QDkhIS9AoEIIRx99tOxVX/R7QQG1Ymr79u2y95133jG1jz/+2NSmTZsm\nH3/mmWea2oIFC2RvU1OTqXln+anVSt6aLfXzegEP9bvwQhQ5oY0cfNICAITB0AIAhMHQAgCEwdAC\nAITB0AIAhEF6sExuuOEGWX/uuecKX+OOO+4wtb/+9a+HfU/AYOEly1QSzUu91dXVmVpVVVXh63qp\nxK6uLlN7//33Ze+//vUvU/v1119N7bzzzpOPP/HEE03tuOOOk70NDQ2m5r2OKpHsJQJVr5cIVK+5\ndw/q9c35vXv4pAUACIOhBQAIg6EFAAiDoQUACIMgRgn09vaa2qpVq2Tv/v37TW3ixImy9+677zY1\nddYPEE3Ol/dqJZHH61VnRnnBhB07dpiaOiMrJR3amDt3rqmdeuqp8vFLliwxNbWuKSV9v2oFU0op\n9fT0mFrOGif1eqWUt6pL3a8XuCCIAQAYkhhaAIAwGFoAgDAYWgCAMBhaAIAwSA+WwJVXXmlq7e3t\nhR9/++23y3pjY+Nh3xMQUc4aJ5Vw81YzqVVFW7dulb3qYMfW1lbZq5x22mmmphKFKaU0YcIEU/MO\nyNy7d6+peWk+dRimerzHS/Ope1OJ6JR00tlLbObgkxYAIAyGFgAgDIYWACAMhhYAIAyCGBnWrl0r\n696KF+Xyyy83tTvvvPNwbwkY9NTKppx1Pl7YQH2pX1NTI3s7OztNbfXq1bL3P//5j6nt2rVL9qpz\nshYvXmxqLS0t8vGjRtk/wfv27ZO96udV51ulpF8zbwWc+v1455KpureSS/GCMl7YRl6jcCcAAEcY\nQwsAEAZDCwAQBkMLABAGQQyH+jL0rrvukr3emTaK+r/lOSMLQ5kKV3hfyKu6t0VBhQ287QybN282\nNS9A9dlnn5naCSecIHsXLlxoarNmzTK1yZMny8erTR2//vqr7FUhCO91zAlHqOuqgEhK+n5znsv7\nXXKeFgBgSGJoAQDCYGgBAMJgaAEAwmBoAQDCID3oeOSRR0xNnbPjueGGG2SdlU2An3pT5zV5K4W6\nu7tNrb+/X/a+/vrrprZq1SrZq9JwixYtkr1LliwxtSlTppial45Tabqcs6y811FdQ50/lpJOXHrX\nVa+vl55Wr6O3kisHn7QAAGEwtAAAYTC0AABhMLQAAGEQxHDcfffdA3r8gw8+KOusbMJwk7PmR60P\n8oIJtbW1prZlyxbZu3HjRlPzzshSK5fmz58ve8eOHWtqBw8eNDW1/iillHbu3GlqXlhB1b1zqFRd\nBTlSSqmnp8fUdu/eLXu7urpMzQu/zJw509TGjRsne3PwSQsAEAZDCwAQBkMLABAGQwsAEAZDCwAQ\nBunBMunt7ZV1bz3KQKlVN14KSa2O8dJNijogM6WUli9fXvgaine/KsnpJaEw+Kgkm5d6U/82vX8X\nKpXo/dtUq4a86zY1NRV6fEopffXVV6bW2tpqat4KJfUzNDQ0yF6V0vP+nqj0n3e45LZt20yto6ND\n9qrfz+zZs2XvpEmTTK2urk725vxd5JMWACAMhhYAIAyGFgAgDIYWACAMghhlos7UKadbbrnF1NQ6\nmpRSamtrM7WHH3645PdUKuq1vOmmm47AnaDcVODBC92odUle7zHHHGNq06dPl73t7e2mtnr1atn7\n2muvmZoKTBx33HHy8ernPfbYY2WvCm2os7BS0kEwL1yhVmdt375d9k6dOtXUWlpaZK+6t5yVXh4+\naQEAwmBoAQDCYGgBAMJgaAEAwmBoAQDCID3ouPbaa03tySefPAJ3UswjjzxSluuqZJG3/ka5/vrr\nZX3x4sWFr3HmmWcW7sXgo1b0qORfSvrfm5eQUyuFvNSuWjWk1helpA+M/PHHH2Xvjh07TE0l5LzH\nq7Sjt+JKve+8lU/qsFmVoExJH2Q5a9Ys2asSl+qwx5T0gY/egZ45+KQFAAiDoQUACIOhBQAIg6EF\nAAhjRCnWahRUsScql2eeeUbWvbN2ilq/fr2sD3S10l/+8hdZ99auKJdeeqmpqfOGBpGBf9OL/1dn\nZ2fh97MKTHjrllTYIOesJS/gsWfPHlP74YcfZK8KYnz77beFe9XPW1NTIx+/a9cuU/PWOKkVSt75\nVGqF27Rp02Sves1UOCMlHRLx/h6MHz/e1LzXQQU0Ghsb5XuZT1oAgDAYWgCAMBhaAIAwGFoAgDAY\nWgCAMEgPYqghPVgBOelBxUsEqjVOXm9OqlAdzKiey+P9nezq6ip0XW/1mVpR5a1xyllxpVYoeT+v\nej7vukpVVVXh3hykBwEA4TG0AABhMLQAAGEwtAAAYXCeFoCKU8GIlPQ6Hy+YoIIY+/btk729vb2m\n5gUT1HW9c6BUQENdV51vlZJea+QFTNS6Je+1OXDggKl5q7PU/XrroQa6sq4U+KQFAAiDoQUACIOh\nBQAIg6EFAAiDoQUACIP0IICKy0nIeSm/nNVMKg3nJQLVyiWVPvSuoX6GnHV53n2p1UpeWlKlEr0D\nMtVrlpPYrLQjfwcAABTE0AIAhMHQAgCEwdACAIRBEAPAoJGzxkn1eudWqbrXq0IX3hqm6urqQvfl\nhStUsOHQoUOyV13DW7ekruu9jt7zDVZ80gIAhMHQAgCEwdACAITB0AIAhMHQAgCEQXoQwKDhpewU\ntRrJW1WkeAdReoclFqXuKyehl7NeKucag2EFUykMjZ8CADAsMLQAAGEwtAAAYTC0AABhjMg55wUA\ngCOJT1oAgDAYWgCAMBhaAIAwGFoAgDAYWgCAMBhaAIAwGFoAgDAYWgCAMBhaAIAwGFoAgDAYWgCA\nMBhaAIAwGFoAgDAYWgCAMBhaAIAwGFoAgDAYWgCAMBhaAIAwGFoAgDAYWgCAMBhaAIAwGFoAgDAY\nWgCAMP4Hl3UbaIcIU6QAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f883c9fc908>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs, \"./my_model_stacked_denoising.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Visualizing the extracted features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_stacked_denoising.ckpt\")\n",
    "    weights1_val = weights1.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure extracted_features_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAakAAABYCAYAAABPlQhTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHoNJREFUeJztndtvXFcVxr8ZX2bsOE7i1E1D04S2oaEXaLmmXHql5SIk\nEK1EQVWrviKe+Dd4BAleKy7iCQkhKkCAEIFCyy30ml6oQx0SJ23qXOw2Hnsyw4P1W2d7nTmO47Se\nTbW+F3tmzpw5Z+119v6+tdbeu9btdhUIBAKBQI6o9/sCAoFAIBCoQgxSgUAgEMgWMUgFAoFAIFvE\nIBUIBAKBbBGDVCAQCASyRQxSgUAgEMgWMUgFAoFAIFvEIBUIBAKBbBGDVCAQCASyRQxSgUAgEMgW\nMUgFAoFAIFsMbtQPPfTQQ+/4IoHdblf1et3+vxjUarXK7/nP+PujH/2otu6LfRvwzDPPrLjYXvfA\ne2Atdqm636pz9gLf6XWsPz+vb7rppr7a89FHH33HfDS1wzuxXmaV/z7yyCN9s+mlPPNVfrse37uQ\n/672zHv84Ac/6Js9v/Od77xtjnMx/d3bgU6nI0nWP58/f16S9K1vfeuC9tywQQqsxQBVTsaN8r7/\nW6/XNTAw0PM7/jXGAv7cva7H30O/UdXZp9frP6u69l7vt9ttSYWtvM1qtZo5G/Akodfve3vmssjx\nWuzpj/Wosjd+2W63zdd4DwwNDUmSlpaWJBUPsrd7r2vIxYa9wHVWPXtS9fV7P1rtWfTnvdh+pNPp\nXLBd+4nV7LfW71bZE1/sdDol+/h79347PDwsqfBbrq/T6VT2Hf4cqyHCfYFAIBDIFhuupNbCSBjJ\nYZKMuozGvG40GpKkkZERex/2z6gOO3399dclSQsLC5KK0Z/P+R6v6/V6iX3koqCAt2V6vesJlwwO\nLrsDdm61WpKks2fPSpJefPHFFefavXv3CttL0ubNmyWttGN67lartapq7Scuhi1zLD7KPeFfo6Oj\nkqTLL79cUmGfZ599VtPT05KknTt3SpL2798vSdq0aZMk6cyZMyv+ep9NfRy7+uvKzbbS6mG2qs/4\n69XlW2+9JWnZv+gHsJ/3X39ufHO1qEkOyulS4FUQ/ucjH/jS3NycpGWbcSz2BNgN4PO+r+W309+6\nFHuGkgoEAoFAtthwJbUa/GjLKP/mm29KkprN5or3+Qtb3bJli7FOWBSsgNEeNspvcVwaR+Wvz2/l\nitWKE2AzVXkg2LpU2AobPv3005Kk559/XlJZ4e7fv1+7du2SJE1MTEgq2mjr1q2SpMXFxRXnzJHh\nV6GXKsHnuK9z585Jko4dOyap8KexsTFJhcpHPR04cEDPPvusJOn973+/JGlyclKS9LGPfUxSoUax\nIb7Lc3D69OkV17fatfcTa8nv+LwQn1XlMHgfpTo4OKht27ZJKnzQqy3aCB/ktVfDvXJ868n/vFPw\nfVSvvDrveXvitz46ha/RLw4ODprCf8973rPiXKdOnZIk/fvf/17xmt8cHx9fce5Go1GKMIDISQUC\ngUDgXYG+KylG6Xa7XWKpsFLPeIiVMvqn1TmwzVdffVVSMYLDPt/3vvfZ70kFWwUpOyHez+/2Ylw5\noFcVU1rxKJXZ+H/+858Vn587d0579+5d8Z6PTXOOa6+9VpJ03XXX2Xl+8pOfSJI+8pGPSJIefPBB\nSYXKJeY9Nzdn15AD218LUjWKH6GMDh48KEmampqSJO3Zs0eSdPPNN0sqbIlqOn78uOVQ8D38nJzf\njh07JEnbt2+XVNj/+PHjkgpbLi4umg19viAHVLXvalMlvDrweSa+gw/Nz89bvtl/BounTVCsqNs3\n3nhDUtEXSOU+JbdnvRdSe/KM0u/5Z3j37t2SpKuvvlpSoaRQpktLSxYNQYnil0RWeAbwR/qA9773\nvZKkG2+8UdKyT/L79KU+z78W9H2Q6pWo5IFjsEJ2csMY8eTJkyu+e/jwYXMyDEaHwLk4lvc53hcJ\njI6OrhoOyBHpYFUVGsCJfZjzjjvu0FVXXSWpsAEPNw898p72ufbaa+07hAQff/xxSdIHP/hBSdJn\nPvMZSUXHMDc3Vyq1zimk0gtc3/z8vI4ePSpJFrL7y1/+Iqno6D760Y9Kkm677TZJ0vXXXy+psN1d\nd91lHQJ2xg60FW1DZ4Df+Q70xIkT9rt0wDwbOdq0V4m/T74zAAM6SsgOnS/HHTt2zMJ9ABvQVlde\neaWkolPFZv55Hh4eNrv5PigHQlUV5kvt6actENqcmZmRJP33v/+VJL3yyiuSyumQyclJC1Xj4z/8\n4Q8lSbOzs5Kk22+/XVJRjMbg5YnaNddcY+3lfTzCfYFAIBB4V6BvSorRHxnYaDRMfsMYYd8wHh8W\nIVEKrrrqKhupr7nmGknFiA0L4Lv8BiEXWO1rr70maVm9cX7PqnwZZy7opfSwB/fLPVAwgUq97LLL\njHWdOHHC3pNkTBVVAINaWloyJfX1r39dUqGksKMv7e90OqUk7v8LFhYWjBFiPwpH9u3bJ0n62te+\nJqkIp8BGjxw5Imm5uAdfQynRJjB/zsl3YLZf+MIXJBWhm7Nnz9o5AO2Zg5KqWrkkZf7cO4qJ+8Ff\nr7vuOknLdpMKm6Duh4eH7Xd4xrEPfQAhLRL+Pow6Pz9v10V/5J+lHKIoVW2aRk24V3D48GFJxTPL\nM02IjtBy2scSFcHvmCKBbfBtnoG//vWvkgp1mxamcA7a1xdfrem+13xkIBAIBAIbjL4pKZ+YbDQa\nVvoIu2LEhiER8yQ+DcMnGXjy5EkbsX25Lr/DOWAfN910k6SCVZHnGh0dtfPD6mBcFxNP3Qj4eHm9\nXjc7EJ/nHmBa2JAkZ7PZLMXfUbYUm8CCsO3x48ct5kzb3X333ZIKloX9se/AwECpLJ7z9htVyyLB\nIMfGxnTFFVdIkr785S9LKq794x//uKSCmaKWyEWleRBKdTkX/n3vvfdKKvIuTzzxhKQil0IkgfaY\nnJy074KcfNPnUHz0pFarmcLmPaInqHb8lbbAr/C3sbExi3iQGyEngv/y+ygon7fhWRkcHLR8Ie2F\nzXNC1SIDzWazVAhGVITnnXunb0Wp8v6hQ4esDehXAaqIPgO7Y9df/OIXK87V7XZL04XWg1BSgUAg\nEMgWG6ak/ORSGB/MJR11qcZhhPb5DdgCFU3EUP/whz/Yef1ESNgVDA1FBcO/9dZbJRXMaW5uzpQT\nx+QQl07hbZlWy8FQ+Yz7grGSuyDWPzQ0ZKyf8/AZqgDGie263a7+9re/SSqYLTkqfg82yzkXFxdN\nidGevjorF/jFMffu3Wvs0lcmUmJOFdUzzzwjqWCy+F2r1TLfxL6U7aN2sQf5g5///OeSVkYdOI7/\n09xuen39hJ/CsdoyRNw7zzQ24hkkn4Qyh9VLRaUaKtaXYWPnD3zgA5KK/gTfpI0GBwdNHaQLrkp5\n5PiArzqmzUdHR81nsBNKlGOwBfeDDZ577jlJy2r0nnvukVS01yc+8QlJRVv885//lFQoen6DSl76\n7TNnzpSiM+vxy3wsHwgEAoGAw4YpqarFUNM8B1Uh5FGoFiHWDDMjps8cFaqiFhYWLMaKcvLLcXzu\nc5+TVLABGD/XwTyXgwcPGgvwC6/moqj8daWL5frPqOZBxcBoYKRpZRMMzc9ngklxjhtvvNHi19gP\nJQG7YlJ1uu0H6tl/NxdU5VLSXJ+P9aMu//znP0uSXnrpJUnSDTfcIKmw8y233GJ+7VUu9n755Zcl\nFVVSsGPyMeShBgYG7DOvAC4lB/B2wSuoXn2A90PmN/r5PNwzVZSpT6Y+LBXqgN+F2eObKFX6BlTu\n0aNH7bvenjnMk6oCftpoNKzduV76Sp45jv31r38tqYiAoFx3795tfSS+hW9TuUuOFHvSb7AYANGG\nl19+2VQrbeDn/K3p/tZ8ZCAQCAQCG4y+bXoIW4WdS8WITG0/lXawVpaaYa4T9fmf//znJS3PiyAu\nDYN96qmnJBWjPnMnyBnA1JhHQEz8jTfesN8n3+CrvXKBX45/dHTU1CgMlJg+MWkYFLY+cuSIxZj9\nHDJew+yx0cLCQqlSD1aLjVAazBfavHmzqSyUVG6AbcJKsesrr7xSqtqDceM/vI/9udcPf/jDkpZz\ndvgzNuF3eJ/8Af6XLoOU/la9Xrf2JG+6nmVn3ml4/0xzRtw7dkLdoMCZF8UKHn7po1arVVplA7/F\n9zkXv09UgT6C17Ozs6ZW+SzHnBTPlld5rVbL7MJnPKvY/Fe/+pWkwpdQSeST7rnnntK8M7/sEb9B\n30pOlVoClNTU1JT9jp+veTHIx/KBQCAQCDj0fZ6UX1RWKhg2FSrkmagc+/3vfy9JuvPOOyVJn/rU\npyQtx7FhQigEzgGz4NywVZQUOSyUwfbt2401wwLIGeTCUnvNj5KWbct9woa4z4cfflhSsaoB+Y9d\nu3YZI/WbmHlmn84o5xhi3xyLUqM9yAOkOancFvGsalfut9VqGbPnfmGNqBvex59RCF7pSMV901Z+\nhROfP/QLfw4MDNh7XDtKJAfm32vzu/T9drtt+Tjsg78SYaGyjJwGc8ew0Y4dO0rbUfAXn8d+voqS\n/BO/3Wg0zMcvlE/LCWl+388LxbfwJebm0T9+6EMfWnH8gQMHrK9EGeFvtAE+TR/i563y/YmJCXte\nsPl6/LL/nhwIBAKBQAX6lpMCaR6FERoFwyhPTojYKIoKhcX7jz32mMXvWUMNBsSoj2LyFVPEZvmt\nbrdbmoOSG6vy15EqIP6niueXv/ylJOmb3/ympPJ6fBMTE8bCqExL84WcVypi/pdddplVVvproE1Q\nTbDVxcXF0tbyuSgpn3+g3bmndrtdmodHDg5b4ZPcI+8fOHBA0rLC5H6Z30eeFBvBevl9GCwKAXbc\nbrcvuDJ2DvCrYKBSlpaWSiskYC/m8fmVY7ARqnNkZMR8jYpANo/kL22ESkMtoXZ7zYP0mx7m4qNS\ncS3ervV63ZQ0FXj4IZEN1n9E4WAzv5lk+hn2wl+xI1v28HxwHP34tm3bSpXC4GIqekNJBQKBQCBb\n9D0nxYjaaDSMMfKXERqmQ/ydz6lcIb46MzOjr3zlK5KK2dCPPvqopIJBpAxMKs/uJ069ZcuWFVur\np9eaQ7xfKud0YC2dTsfmJ/h5O+ROUJZ8Z2FhwarN/vWvf0kqVAHHYneYabqiOXaDbQHyhRx3+vRp\na7fcclJV7Zra18/vY6Y+vgmTxWep2IN9zszMlDbfQyVQheZXW6DNyKmQ9zpy5MiKNucapbzW8ANe\n3Q0NDRlzx27YCR8j8oFtWDeOtvrtb39raos5VEQHUGNESWhH8rDYCJY/MjKSrcpP4X0QtNtti0JR\nzYtdyf199rOflVTMaeJ5RXnt27fP/I78H+dinVP6ZfaM4xz0MeRpO51Oqc3X45959LaBQCAQCPRA\n35VUur0woy5znYibUqUDy2EEhznBAr7//e/rvvvuk1TMB/jHP/4hqcg1kUOBscHMiP/D2CYmJkxd\nwapy26HX58jSXU5RUtw/90DcGNX0wgsvSFqu0EENUNXDWmdpPkkqmOn09LS1lZ8fxTl8O8/Ozpbm\n8uSiTD08C9yyZYuxdhQAvsl8Je4bpur316rVahbr5zuAyivyqkQIsD/Klpj/qVOn7Fyoq147XfcL\nVdfA+81m0/K+RC28L9MHoIrwJxTC+fPn7Tllrg8r1GNzv+oJtupVccZnOcIrJ7/bdr1et+cPtY6S\nP3bsmKRCHVGxh33p23bt2mXHoqiwCXNNWZGCPpVoAnNPeQbGxsbsuafvWM+zvuELzPrl5flbr9fN\nUDiXXxKFBDOdJ2ESwiAPPPCAOT3JVKQ/hmOxTkIofnNEQgFDQ0M2KFKC7JfJ6Tdw2tSG0rLT4miE\n6u666y5JRYgJp2FgevPNN80ZSTjj/LSD39p7bGysVIL+xS9+ccWxvE9bjY6O2nlpq1wG/QuFczdt\n2mSTPLkfvkOnSYdJWS6dK+GWbrdrYROW9eIzQtEMQn6RUAY1woGNRqO0oKxfGLWfqHrmwdatW+26\nCRcTrsI/IUHcF4SUjnF4eNgKqb761a9KKmz/ve99b8W58UXaiP4G4rF58+bSwOXTEv2E90tv34WF\nBRuU0i3cpaI//NnPfiapeO7pH/h76tQp8zfOQZj/N7/5jSTp/vvvl1T4JQMiwN6Li4tGDPwzH4UT\ngUAgEHhXoO8l6Iy0586dM6YDO0eS+g38CLXAphjRz5w5YwvHksSD2X/pS1+SVLArroMySpJ/KICR\nkRFjWn55nNzDU61Wy9Qo0hz2jkTnnmBJjUajtBglYRJkPyyXEOnCwoKxWGyFKoBd+YVum81mSUXn\nwFKlaqaK+kzLx2HcMFd8g7+wcxQB57jhhhtMCaHusS9qjM/xd0CbYtN0Y86ct5bwSp9rHR8fN3/h\nGSfCgarENtjRbydz5ZVX2pJT+DRqCztiN0LbJPZh/GBoaMjC1j5qkova7wWudXx83CIoPI+PPPKI\npMJfiUqh0tn8ELu+9NJLpWeXaACbctJHPvnkk5KKZ50ydxT/uXPnSotW++1v1nR/az4yEAgEAoEN\nRt+26uA1+Z7LL7/cFBRskIQxIzMMns+Z7Mu5pqenLc5Pko9cDOwKpkGuCobk4/+bN2+20Z9rhAHm\nwqqqFMnp06ft2lGSKEbYOTFq7LF//36zN+w23ahQKgpW0sml2M1vs0Lsme+S15qfn89GOV0IXCcq\naHJy0hL83C9g0WN82G9NkJ4Du7LlvN8CHtuRj4G5UoRxyy23SFq2P8w/x23Oq7bnSZUghTwop3Qr\n9xSoHtTRoUOH7BxEQciZ+AVmsT2q3+dladPJyUnra1BwuTzrKbAj/RE5tp07d9qi2ygnlpW6/fbb\nJZXVO88r/eXU1JSdH3tRA4Dawq74JeqXYiG+d+jQIetPac/1KNNQUoFAIBDIFn0rQfcb9S0tLZW2\nQ/Y5IFgWpbrE8BnJjx49aqM8LBS2xLJAMGCYGyXpHEd8dceOHbYUva/4yaW6zyPd/gSGSjyZz1BQ\n09PTkgp29MQTT5TK9GkHX12GvQ8ePGjL9/v8Ce3qF+ntdDr2WW6TeT165Xco2aUMH3D/3K9fMBWf\nefzxx82eME6AXZkAjS8SHYDx//3vf5e03B5+8U+uNQe1WtWuveyKEiTHhl/yGntSQk3e9K233rLz\n/fGPf5RU5Fn4DsqV5xg7+u15ms2mHeunm+Tgoz63x3OELQYHB02xkJNHZdLf8Uzjr/SHf/rTnyQt\n9x3kU+kPqPwj58SxtAWKCkVHhGt8fNwiLX5x7ovpQ0NJBQKBQCBb9K26j5E0rQrzy8AwosN0qNIh\n7spk3nR7aNgpv8OxjP7U/sNwUQ2M9LCB6enpUg4qt6VSvC35Ozc3Z4yK+4NV+sm2qVpFmTK3wm9W\nx+/BREdHR03VwoRhqyhT3ue4Wq1WWnQyh63Oe4H7xy9nZmaMoXLNn/zkJyUVvsH9k4Pzmz7Ozc1Z\ndZnf1hw7+3wW7cEcvnSCpt96BpuuZ3O5txtVcyL5u7S0ZDk13mMJHvyUXDJKgNwq991sNo3J42u0\nEc86tqH/IMpAFABbjYyMWJ9TtTBqP1FVIZdWTZJ74t6x4+9+9ztJhW+hjugfiAzs2bOnFCVAzfpo\nAX6Izeh708VqyQ9eyiTzUFKBQCAQyBYbrqT8SJqy9JRhSUWslZHaL2RI7DldTobRnmorGCzM4tOf\n/rSkYg4Qoz7xVhjykSNH7Fw+35ILvC259vPnz9tnftND5kuhpFCpp0+ftvyVX+wTJgUDTSvYYFMs\nz8NnVPqgqGCru3btMnaXq4ICfqvyqamp0kaE+A8MlaVhYL0oWD7ftm2b5UtRRj/96U8lSXfffbf9\njlTkC1ByfuO/2dlZUxa0EefOCV6Rg/n5ebtHloBCWaF6UFKoJPqEdLFjclK0DecgD01VKpWsfvM+\n2ujMmTOlds1p3hn2S7c6kYr8cr1eNx/Crj6PRJ6de0eFovgbjYblA7n3H//4xyte03dQIY36xffI\ne6db/fj5UheD/ls+EAgEAoEK9K26jxGVEbbZbJa2wmbkhhWwwKZfZw92sHPnTmNcbDcBw3zggQck\nFWyKvBfVUSiKdMa2X8o/Bza1GtJ8E9eOWknX5pIKRkq8fvv27aUcV7oaiFQwTpjbyZMnrc1gwsT5\nqxbzhO3+P4B2Rw2mc7xQmdgCf/JzSWC25Dpfe+018zlUKLbiHCgr3r/55pslFfkDbH3FFVeY/bFv\nTswf+K1k8KP5+XnzP54/vx4czzaRDyrHePbr9bpty4PSRx3QX3AOoil+Q0XsvrCwYL7Ps5TTlic+\nJ0XEg/uempqyfCeVuKgt7hGfYwsf1BDHT01N2Vqo+D99IuciakKuFBXGs4Dfpit4XMpi0vl4ciAQ\nCAQCDn2fJwXLarVapRWlYZyM3MTfGZ1hW3xvaGhITz/9tKRCQbFiL2yLXBWMlzgqLCUd+WGnvpov\nl+o+j3SlDhQNNvKxdtQBNk63nPfqErZLO8BqpUI5ocx8Hi9ds09aZlleKecy76yqWjKt/ITpwx5R\nTORQyHfA2lGS6WrQsHTOxaoggLlY2B+mTH4GNTE2NmaqAHWSk4KqAso8fcZg9H7FCV8FTB/A3LHd\nu3dbrhpb0xbYl5VqyGGjoLAn1zM6OmpVgyAX35Sq15bkOTpx4oStzEG/xrPK+yjDxx57TFLR3xFp\nOXz4sNngzjvvXHEs6pUNZPFP7Mm6qbTVnj17VlROptd8Ufd90d8IBAKBQGCD0PdND1M2wPwbGD2f\nwYxgTLBHWA6VZHv37rW1+jwTIwZL3JSafp9DgRk3m83KFXtzYVeelXDPtVrN1A8skdccA/OGhc3N\nzZl9YfKcn4pAZo9j73379hlDYu5Euop8en6UW7vdrlQs/UbVvB6U1MTEhM37QDlyf1SRogTwUfJH\nrDU3NjZmKoxN+mCbVPX5VRf4nCo1MDw8nFXOpAq+ndPcCvbiucS+3LOfI5buVSQt74TAs+0360xX\nYpAKv2ZeGj6a7s/l1X1OG3P6Nfu4pnTFDOyJ3ciFUmnH+o+33nqrpOU1O6XC3k8++aT1w6lalYo5\ngXfccYekog04J31oOofNry6znrX7+j6ZNy1P9Qt6ciyhAAxJR8hAgyFmZ2etAf3maZwTB+YcJAxJ\nKKZLjuQa1gNVD9Lg4KDdNwML98drHJsQyNatW3XbbbdJKmzGw43jsxwPtu12u+b8vvOgLWkzHLjb\n7ZZKaXODtyvFOHNzczbY+0Ee/yJUzXewIa+ff/55OweLfzJRFRtSIOF9lNAMoVPapde15wD//Phr\nW1paMrtw70wvgVwx2POsM92EkNP4+LiFQz3JIvRPOIpQrE8TpDtaX2hjwX7C7xDONfGM7dixw3wF\n32KwghB9+9vfliR997vflVQQIZ7jgYEBG8C+8Y1vSCp8mAIKwqSE//2CtPjpyMhI5e7hsVVHIBAI\nBN4V6Ptk3nSkrVqCnhEaBuq3fYZRnj171pg8oznMAsC6SKr6ZefTLdmrwj85sKoUvRirL1iARfKa\nkCo2HBgYsO8Q/oBBIf/9hn4zMzNmC5Sa30oaptzLrlXXnxtSf8O/KOmFtRPWg21iZx9uabVa1hbY\nlb9efZH89j6MKq3X61ksJFuFCy2LlLY7foOCwubcOwqLQgDCp5s2bSr5E0yfZ5s2wO5VU0vS68mx\nlB+spvbwDfzQh1gJ2VGeTwiZEOirr75q4X3SLHzGJHJ+wy8XR+FKugSbv9YoQQ8EAoHAuwp9K5wA\nKWPxy5v4RUphPn7LaRTWxMSEsSeUAq/9BoZ87uPSKSu7UEy936hKRna73VLSH3ZD/NpPWmy328b+\n/eaT2IxjYaZLS0vGSmG8fiKwR87Mv8qevZgqIJ/EMSSZUfK+RLxWq5nNyMPgz5zD+7C/Lt7vdrvZ\n+WQKb0eewTTx798jwsEzzTkocPI26DWlgYIJpp3gz/i3X5ar1zOfu7qXyn7R6XSs70RxEuGoUuso\nUqbmvP766xYlIKJC3oo24i/25DVttp4t4ldDKKlAIBAIZIu+K6mUsfj4abqNRwq/+COKqlarGYuC\nJZEL8EoprWb5f4Vn0elrv6gnrIcqNF/B1ul0jHXx16tNv2R/usW3X5YH+Nh+yq5yy/FV2TO95rQS\nLH1NPN6rBxgsbDSNGPjtIHyloJ9Y2kuF5mbDFFVTJFL454/veMXq308r3apynKgyn3Pyywv1st2l\nlExvFHr5K/bB36r8guNQXvjvli1bzB9pGz8twEdLOM5HpXopqajuCwQCgcC7CrUcGVggEAgEAlIo\nqUAgEAhkjBikAoFAIJAtYpAKBAKBQLaIQSoQCAQC2SIGqUAgEAhkixikAoFAIJAtYpAKBAKBQLaI\nQSoQCAQC2SIGqUAgEAhkixikAoFAIJAtYpAKBAKBQLaIQSoQCAQC2SIGqUAgEAhkixikAoFAIJAt\nYpAKBAKBQLaIQSoQCAQC2SIGqUAgEAhkixikAoFAIJAtYpAKBAKBQLaIQSoQCAQC2SIGqUAgEAhk\nixikAoFAIJAtYpAKBAKBQLb4H5wVmOwonlIoAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f883c724eb8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i in range(5):\n",
    "    plt.subplot(1, 5, i + 1)\n",
    "    plot_image(weights1_val.T[i])\n",
    "\n",
    "save_fig(\"extracted_features_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Sparse Autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ageron/dev/py/envs/ml/lib/python3.5/site-packages/ipykernel/__main__.py:3: RuntimeWarning: divide by zero encountered in true_divide\n",
      "  app.launch_new_instance()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure sparsity_loss_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmcjXX7wPHPNTKYDEaN7Ma+hZBKZUlpoWwpIsnyPHqS\n6kGFNku0/RCylFLTQhKKaLWTtZAlWce+NoxlBrN8f398z8wzM2bGrOc+y/V+vc6rzjn3uc917jHn\nmu96iTEGpZRSytMEOB2AUkoplRZNUEoppTySJiillFIeSROUUkopj6QJSimllEe6xukAcpOI6JRE\npZTyIMYYye5rfa4FZYzRWxq3119/3fEYPPmm18d/rs306YYWLfT6uOOWUz6XoJRSKiPjxsEzzzgd\nhcoMTVBKKb+xZg2cPAlt2jgdicoMTVB+onnz5k6H4NH0+qTPl67N2LHw7LOQL1/undOXro+nkdzo\nJ/QUImJ86fMopXLPgQNQvz5EREBwsNPR+AcRweRgkoRPzeJLT1hYGPv373c6DOWQChUqEBER4XQY\nymHvvw/du2ty8iZ+0YJyZXEHIlKeQH/+6vx5CAuDDRvsf5V75LQFpWNQSimf9+mn0Ly5Jidv4xdd\nfEop/5WQYKeWf/KJ05GorNIWlFLKpy1YAMWKwR13OB2JyipNUEopnzZ2LPz3vyDZHglRTtEE5aPC\nw8Np0qRJ0v3g4GCdyab8zu+/w+7d8MgjTkeiskMTlMMqVqzI4sWLk+5/9dVXFC9enBUrVrB//34C\nAgJISEjI1rkl2Z+M586dI0xHiJWfeftt23rKn9/pSFR2aILyIOHh4fTr148ffvghqfUjHtwvER8f\n73QISqVr925YsgT+9S+nI1HZpQnKQ3z44Ye88MIL/Pzzz9x6661Zfn1kZCRt2rShaNGi3HbbbezZ\nsyfF8wEBAezdu5e1a9dSqlSpFOuC5s6dS7169QC7G/xbb71FlSpVCA0NpXPnzpw5cwYgqUU3bdo0\nKlSowN133w3AZ599RlhYGKGhobzxxhspWoWZOd9nn31GhQoVKFGiBKNGjUqKKyEhgVGjRlGlShWK\nFi1Ko0aNOHz4MAA7duzg3nvv5brrrqNmzZrMmjUry9dM+bbRo+Gpp6BwYacjUdnm9Hbsuby1u0lL\neo97grCwMPPwww+bkiVLmi1btqR4LiIiwgQEBJj4+PirnqdTp06mU6dOJiYmxmzdutWUKVPGNGnS\nJOn5gIAAs2fPHmOMMVWqVDG//vpr0nOPPPKIeeedd4wxxowdO9Y0btzYHDlyxFy+fNk89dRT5rHH\nHkuKR0RM9+7dTXR0tLl48aLZvn27KVy4sPntt99MbGysGThwoAkMDDSLFi3K9Pn+/e9/m0uXLpnN\nmzebAgUKmB07dhhjjHnnnXdM3bp1za5du4wxxvz5558mMjLSXLhwwZQrV86Eh4ebhIQEs3HjRhMa\nGmq2b9+e5rXx5J+/yhvHjhkTEmLM8eNOR+LfXL972f9Oz8mLPe2W3QQFuXPLjrCwMFO0aFHTrl27\nK57LbIKKj483+fPnNzt37kx6bMiQISkSlIgkJahXXnnF9OzZ0xhjzNmzZ821115rDh48aIwxpmbN\nmmbx4sVJrzty5IjJnz+/iY+PT4onIiIi6fnhw4ebLl26JN2Pjo5OkaAyc74jR44kPX/LLbeYmTNn\nGmOMqV69upk/f/4Vn3fmzJmmadOmKR7r06ePGT58eJrXRxOU/xkyxJj//MfpKFROE5R28ZFb6Sn7\n7z9lyhR27txJr169svX6kydPEh8fT9myZZMeq1ChQrrHd+nShblz5xIbG8ucOXNo2LBh0mv3799P\n+/btKV68OMWLF6dWrVrkz5+f48ePJ70++fscOXKEcuXKJd0vVKgQ1113XdL9zJzvhhtuSPr/oKAg\nzp8/D8DBgwepVKnSFfHv37+fNWvWJJ0zJCSE6dOnc+zYsUxdL+Xbzp2DDz6AAQOcjkTllCYoD1Ci\nRAkWLVrEihUrePrpp7P8+tDQUK655hoOHjyY9NiBAwfSPb5mzZpUqFCBhQsXMmPGDLp06ZL0XPny\n5fnhhx+IjIwkMjKS06dPc+HCBUqVKpV0TPKJG6VKleLQoUNJ92NiYvjnn3+ydL70lCtX7oqxtMTH\nmzdvnuKcZ8+eZeLEiVc9p/J9U6fC3XdD5cpOR6JyShOUhyhZsiSLFy/mp59+on///kmPG2O4ePEi\nly5dSrqZVM21gIAAOnTowNChQ4mJiWH79u2Eh4dn+H5dunRh/PjxrFixgkeSLRLp06cPQ4YMSUpw\nJ0+eZN68eSniSa5jx47Mnz+fNWvWEBsby+uvv57i+ayeL7nevXvz6quvsnv3bgC2bNnC6dOnefDB\nB9m5cydffPEFcXFxxMbGsmHDBnbs2JHhZ1a+7/JluzD3xRedjkTlBk1QDkveGilbtiyLFi1i9uzZ\nvPzyy0nPBwcHExQURKFChQgKCmLJkiVXnGfChAmcO3eOUqVK0bNnT3r27Jnu+wB07tyZZcuWcffd\nd1O8ePGkx5977jnatm3LvffeS9GiRbn99ttZt25duuepVasWEyZMoFOnTpQuXZqiRYtSokQJChQo\nkK3zJb/fv39/Hn300aTX9u7dm5iYGAoXLszPP//MV199RenSpSldujSDBg3i8uXLGV9s5fNmzIDq\n1aFhQ6cjUblBy22oXHXhwgWKFSvG7t27MxwHcyf9+fuHhASoUwfeew9atnQ6GgVabkN5gO+//56Y\nmBguXLjAgAEDqFu3rsckJ+U/5s6FoCC45x6nI1G5RROUyrHvvvuO0qVLU7ZsWfbs2cNXX33ldEjK\nzxgDI0bAa6/pprC+RLv4lM/Tn7/vmzcPXn8d/vhDE5Qn0S4+pZRfS2w9vfKKJidfowlKKeXVfvoJ\nYmKgfXunI1G5TROUUsprGQPDh9vWU4B+m/kc/ZEqpbzW4sUQGakFCX2VJiillNcaMQKGDIF8+ZyO\nROUFTVBKKa+0fDkcOgTJtpJUPsaxBCUiISIyV0TOi8g+EXksneMCRWSKiBwTkVMi8p2IXH2nUS8R\nFhZGwYIFiYyMTPH4TTfdREBAAAcOHODw4cN07NiR0NBQQkJCqFevHp999hnwv6J/RYoUoUiRIgQH\nB1OkSBEt4Kd83ogRMHgwXHON05GovOLkj3YScBEIBRoAC0RkkzHmr1THPQ/cCtwInAU+AiYAHd0Y\na54RESpWrMiMGTPo27cvAFu3buXixYtJ+9J169aN+vXrc/DgQQIDA9myZUuK0hIiQlRUlEeXh1cq\nN61YAXv2QLduTkei8pIjLSgRCQI6AK8YY2KMMauAeUBa/9zCgJ+MMaeMMZeBr4DabgvWDbp165Zi\n9/Hw8HC6d+8O2N2+169fT/fu3SlYsCABAQHUq1eP++67L8U5dCGq8hfG2Fl7r70GgYFOR6PSZAxk\no3RQak518VUD4owxyYv9bCbtxPMxcKeIlHIltq7AQjfE6Da33XYb586d4++//yYhIYGvv/6axx9/\nHLCto8aNG/P0008zc+bMFDWfktMEpfzFokVw7Bi4fkWUJ3rvPVi1KsencSpBFQaiUj0WBQSncexO\n4ABwGDgD1ABG5Go0Q4faJeipb0OHZv749I7NpMRW1C+//EKNGjUoXbp0UtKZNWsWTZs25Y033qBS\npUo0aNCADRs2JL3WGENoaGhSddnixYvz999/5ygepTxRYutp2DAde/JYP/wA775r95/KIUf24hOR\nm4CVxpjCyR7rDzQzxrRNdeyXQEGgJxANvAQ8aIy5LY3zmuQF85o3b07z5s09ei+2ihUr8vHHH1Ol\nShWaNm1K48aNefDBB+ncuTP58+cnIiKC8uXLJx0fGRnJgAED+PXXXzl48CD79++nUqVKxMXF6RhU\nOjz556+y5vvv7cSIzZt1Ya4nWvrppyzt2xc6d4Zy5Rg2bFiO9uJzKkEFAZFA7cRuPhEJBw4bY4ak\nOnYLMMQYM991vyhwGrjeGBOZ6liv2yw2MUG1aNGCu+66i40bN3L06FECAwPTTFAA27Zto27dupw6\ndYqzZ89SqVIlYmNjCdDf2DR58s9fZV5CAtx8sx17atfO6WhUmtauhb174TE7KdsrN4s1xkQDc4Dh\nIhIkIncAbYDP0zh8PfCEiBQRkfxAX2wii0zjWK82bdo0Fi9eTKFChVI8PmjQILZt20Z8fDznzp1j\n0qRJVKlShZCQEMB28ekXsPJ1c+faVlPbtlc/Vjnk1luTklNucPJP7r5AEHAC+BJ4yhjzl4jcKSJn\nkx03ELgE7AKOA/cDPrMtZPJuuYoVK9KgQYMrnouOjqZ9+/aEhIRQpUoVDh48yLxk/bsiQkhISIp1\nUO+99577PoRSeSw+3racRozQHcv9idaDUj5Pf/7e78svYdIkWLlSE5Q3yWkXnyYo5fP05+/dYmOh\nVi344ANo0cLpaFQKp07B9den+7RXjkEppVRmffQRVKyoycnj7NwJN94I6azNzA26kkAp5bHOn7f1\nnhb61NJ8HxAZCQ89BG+8AeXK5dnbaAtKKeWxxoyxLaf69Z2ORCW5fBkefhhat4bevfP0rXQMSvk8\n/fl7pxMn7NjT+vW2i095AGOgZ0/bgpoz56qFuHI6BqVdfEopjzRihN1vT5OTBzl+HP75B2bMcEuV\nSL9oQYWFhbF//34HIlKeoEKFCkRERDgdhsqC3bvhttvgr78gNNTpaFR26TTzZNJLUEop79K5M9Sp\nAy+/7HQkKic0QSWjCUop77d+vd1rb+dOuPZap6NROaHroJRSPsMYGDQIXn9dk5NHiI119O01QSml\nPMb338PRo3aimHJYbKydSp4LdZ2yS2fxKaU8wuXLMHCgLcaqxQgdZgw88wzkzw+tWjkWhv4zUEp5\nhMmT7ZTyBx5wOhLFmDGwZo3dndfBvxZ0koRSynGRkVCjBixZArVrOx2Nn5szB559FlavzvE2RjqL\nLxlNUEp5p+ees0MekyY5HYmfi4uDJk3sDyIX9pfSBJWMJiilvM+OHfY7cft2XZTrERISbOniXKDT\nzJVSXm3gQDu1XJOTh8il5JQbdJKEUsoxv/xiW1CzZzsdifJEnpMqlVJ+JS4O+veHd9+FAgWcjsZP\nJSTYKeUeShOUUsoRkyZBiRJ2WyPlAGPg6adhyhSnI0mXTpJQSrnd8eO2WviyZbbmk3LAm2/C11/b\nH0KRInnyFloPSinldQYNgu7dNTk55osv4IMP4Lff8iw55QZNUEopt1q9Gn7+2dZ6Ug5YvBgGDLD/\nLV3a6WgypGNQSim3iY+3W7y9845H/+Huu4yBN96wXXtesGWHjkEppdxmyhSYPt0Oe0i2RyZUjuTi\nQtyr0Z0kktEEpZTnOnXKjjn9+ivUret0NModNEElowlKKc/Vpw8ULAjjxjkdiXIXncWnlPJ4GzbY\nunc6McLNYmNtuQwv7U/VSRJKqTwVH2/Xg44aBcWKOR2NH0lIgCefhPffdzqSbNMEpZTKU5MmQaFC\n9rtSuYkxdir5gQPQu7fT0WSbdvEppfLMoUMwbJgtzOqlvUze6e237WyU5cvtXwdeShOUUirPPPcc\n9O1rq+UqN/nkEzuff9UqCAlxOpoc0QSllMoT8+bB1q3w5ZdOR+JH4uNtyfYff4QyZZyOJsd0mrlS\nKtedO2c3KggPh7vucjoa5RRdB5WMJiilPEP//hAZCZ9+6nQkykm6Dkop5VH++MN2623b5nQkytvp\nNHOlVK6Ji4N//9tuBnv99U5H4wdOnLDjTj5KE5RSKtdMmADBwfDEE05H4gdOnICmTWHhQqcjyTPa\nxaeUyhW7d8PIkbBmja55ynNnzsB990GnTvDQQ05Hk2cca0GJSIiIzBWR8yKyT0Qey+DYBiKyTETO\nichREennzliVUhlLSLAbFrz8MlSp4nQ0Pu7CBWjd2raehg51Opo85WQLahJwEQgFGgALRGSTMSbF\ndpIich3wA/Ac8A1QACjr5liVUhn44AO4dAmefdbpSHzcpUvQvj1UrQpjx/p8U9WRaeYiEgScBmoZ\nY/a4HvsMOGSMGZLq2JFAWWNM90ycV6eZK+Vm+/dDw4awYgXUrOl0ND7u7FkYMwZeecXuUu7hcjrN\n3KkuvmpAXGJyctkMpFWD+DbgtIisEpHjIvKdiJRzS5RKqQwZA//6l92XVJOTGxQpYrv1vCA55Qan\nPmVhICrVY1FAcBrHlgXqA/cAW4F3gRnAnWmdeGiyPtnmzZvTvHnzHAerlErbtGl2Qe4LLzgdifIE\nS5cuZenSpbl2Pqe6+G4CVhpjCid7rD/QzBjTNtWxm4DfjTG9XPeLA6eAosaYc6mO1S4+pdzk8GG4\n6SZYtEhLuKu0eWsX307gGhGpnOyxekBaa8//BFJnHQP49uigUh7MGHjqKbtTuSanPPTNN3D5stNR\nOMaRBGWMiQbmAMNFJEhE7gDaAJ+ncfgnQHsRqSsi+YFXsa2vs+6LWCmV3Gef2ckRQ4Zc/ViVTe+8\nYydDRKUeDfEfTu4k0RcIAk4AXwJPGWP+EpE7RSQp+RhjlgBDgIXAMaAS0MWBeJVS2MQ0cCB8/jkE\nBjodjY8aNw4+/ND2n4aGOh2NY3Q3c6VUpiUkwD33wL33wqBBTkfjo6ZMgbfegmXLoEIFp6PJEd3N\nXCnlNuPH27WiOmsvj8yfD6NGwdKlXp+ccoO2oJRSmbJ9u91dZ+1aqFz56serbLhwAY4fh0qVnI4k\nV2jBwmQ0QSmVN2JjoXFjuyi3Tx+no1HewlunmSulvMgbb0CJErbWk1LuomNQSqkMrVtnx+03bfL5\nvUmVh9EWlFIqXRcu2OKDEyZAqVJOR+NjFi6Enj2djsKjaQtKKZWu55+HW2+FRx91OhIf8/33NjnN\nn+90JB5NE5RSKk3ffANLlsDGjU5H4mPmz4devWySuuUWp6PxaDqLTyl1hQMH4OabYcECaNTI6Wh8\nyLx5tvSwn1xYncWnlMpV8fHw+OO2xpMffIe6jzHw9dd+k5xyg7aglFIpjBhhNzL45RcI0D9hVQ7o\nVkdKqVzz228wcSL8/rsmJ+W8TP8TFJESIjJORHaLyEUROSgiC0TkgZwGISIVRCRBRBrk9FxKqeyJ\nioKuXeGDD6BMGaejUSqTCUpEKgAbgZbAS0AdbAn2hcDkXIhDuLIooVLKTYyxu0Q88AC0bXv141Um\nLF0KMTFOR+HVMtuCmoxNIA2NMbONMbuMMX8bYyZiK+EiIuVEZK6InHXdZotI0t9hIlJWRL4VkX9E\n5IKIbBeRxNUVe13/3eBqSS3OrQ+olLq6yZPh779h9GinI/ER06fDY4/Z4lkq2646BiUiIcB9wBBj\nzBV/DhhjEss9fgdEA81d9ycCc4HEif6TgUCgGXAOqJ7sNLcA64B7sSXe/bfGsVJu9vvv8Prrdvyp\nUCGno/EBH31kL+ivv0KNGk5H49UyM0miCrYLbkd6B4hIS2y3XyVjzEHXY12A3SLSwhizGCgPfGOM\n2ep6WfI/LU66/htpjDmRxc+glMqmqCi7S8TEiVC1qtPR+IDx420zdOlSvaC5IDNdfJmZIlgDOJKY\nnACMMfuAI0At10PjgFdF5DcRGaETIpRyljF2t53779etjHLF7Nk2QS1frskpl2QmQe3Cjj/VzOCY\njCY5GABjzDQgDJgGVAV+E5HXMh2pUipXTZgAEREwZozTkfiI1q1h5UqthJuLMrVQV0QWAnWBasaY\n6FTPFQUaAT8AlY0xB1yPV8Imt3uMMUvSOOeLwLPGmLIiUgo4DNxqjFmf7Q+jC3WVypT16+336erV\nWh1X5R13bXX0NLaVtEFEOopINRGpLiL/ATYbY37FTm74UkQaiMjNwBfAhsTkJCLvich9IlJRRG4C\n7ge2uc5/AogB7nOttyqS3Q+klMrY6dPQqZOduafJSXmyTCUoY0wE0AD4BXgL2AwsAh4EEmtstsVO\ndljieu4I0D7Ve43HJqWfgGPAk67zxwP9gN7YltS32f5ESql0JSRAt27Qpg08/LDT0Xix+HiIjr76\ncSpHdC8+pfzI0KGweDEsWgT58zsdjZe6fNlm+cqVYdQop6PxaLoXn1IqU77/3i7R2bBBk1O2RUdD\nx472Ar6mc7zymm4HqZQf2LXLTimfNQtKlnQ6Gi91+jS0bAmhoXZKecGCTkfk8zRBKeXjzp+HDh1g\n2DBo3NjpaLzUqVPQrJmtgPvJJ3CNdj65g45BKeXDjLFbwhUqBNOmgWR7NMDPxcbaVlOnTnoRsyCn\nY1CaoJTyYWPGwJdf2vWjus+ecjedJKGUStOSJfDOO7B2rSYn5Z10DEopHxQRAV26wOef68472aI9\nMR5BE5RSPubcObsQd9AgO+lMZdHMmXZWiXKcdvEp5UMSEuDxx+HWW+HZZ52OxguNGwf/93+wYIHT\nkSg0QSnlU155xS7XmTVLJ5tlSUICvPSSTUyrVkH58k5HpNAEpZTPmD4dvvrKTooIDHQ6Gi8SGwtP\nPmnLs69cCcWLOx2RctExKKV8wLp18Pzz8N13dqMDlQX58kGjRvDLL5qcPIyug1LKyx0+bMecJk2y\nkyOU8hTuqgellPJA0dHQrh3066fJSfkeTVBpCAgIIF++fAQEBFxxy5cvHz179nQ6RH766ScCAgKI\n1po0fis+3s7Yq1kTXnzR6Wi8iPayeA3HEpSIhIjIXBE5LyL7ROSxqxyfX0R2iMiBvI7t2LFjHD16\nlGPHjjF16lREhOPHjyc9Pm7cuGydNy4uLtdiNMYkNp9z7ZzKu7z4op2x99FHOmMv0+bOhfvu0yTl\nJZxsQU0CLgKhwOPAZBGpmcHxL2Kr8Oa5EiVKJN2KFSsGQGhoaNJjwcHBAAwYMIBq1aoRFBREpUqV\neOWVV1IkocGDB9OoUSOmTp1KpUqVKFiwIAkJCZw7d44uXbpQuHBhypQpw5gxY2jZsiVPP/100msv\nXbrEgAEDKFu2LIULF6Zx48YsWbIEgL///ptWrVoBEBwcTL58+VK8Vvm+99+HhQthzhydsZcpxsDo\n0bYv9K23NKN7CUemmYtIENABqGWMiQFWicg8oBswJI3jKwJdgP7AVHfGmpFixYrxxRdfULJkSbZs\n2UKfPn249tprGTx4cNIxO3bsYN68ecydOzep27Bfv36sW7eOBQsWEBoayquvvsr69eupWrVq0uu6\ndOnCqVOnmDVrFiVLluS7776jVatWbNq0iWrVqjF9+nS6du3Kvn37KFSoEEFBQU5cAuWA77+3hVxX\nrYKQEKej8QJxcTYxrVoFq1dDuXJOR6Qyyxjj9htwE3Ah1WMDgO/SOX4+0AZoBhzI4Lwmt33zzTcm\nICAgU8e+9957pk6dOkn3Bw0aZAoVKmTOnDmT9FhkZKS55pprzLx585Iei4qKMsHBweY///mPMcaY\nbdu2mXz58pkTJ06kOP/9999vBgwYYIwx5scffzQBAQHmwoUL2f5syvts2GBMaKgxa9c6HYmXiIkx\n5v777S0qyulo/I7rOznbucKphbqFgahUj0UBwakPFJH2QD5jzDwRaeaO4DJrxowZvP/+++zdu5fz\n588TFxdHgQIFUhxTsWJFihYtmnR/165dJCQk0KhRo6THihQpQo0aNZLu//HHHyQkJFC5cuUUY0yX\nL1+moFbx9FsHDkDbtvDBB7ZunsqEAgWga1fo3FmLDHohp35i54EiqR4rApxL/oCrK/Bt4IHEh652\n4qFDhyb9f/PmzWnevHkOwkzfsmXLeOKJJxg1ahR33303RYsW5euvv2bEiBEpjrv22mtT3E9MOJJB\nH3hCQgKBgYFs2rTpiudSn0/5h6goaNUKBg6E9u2djsaLiNipjsotli5dytKlS3PtfE4lqJ3ANSJS\n2Rizx/VYPWBbquOqAhWAFWK/0QOBoiJyBLjNGHPFjL7kCSovrVq1iipVqvDCCy8kPbZv376rvq5a\ntWoEBASwbt06HnroIQDOnj3Ljh07uPnmmwFo0KABsbGxnDx5kltvvTXN8wS6Rsbj4+Nz+lGUh7t0\nyW6ufddd8NxzTkejVPpSNwqGDRuWo/M5MovPGBMNzAGGi0iQiNyBHWP6PNWhW4By2DGrekBv7Ey+\nesBB90V8pWrVqrFv3z5mzZrF3r17GT9+PHPmzLnq60JCQnj88cfp378/y5YtY9u2bfTq1Yt8+fIl\ntapuvPFGOnToQNeuXfn222+JiIhgw4YNvPPOOyxw7bIcFhYGwPz58zl16pSuh/JR8fHQrZudDPHe\nezr5LEPGwJkzTkehcpGT08z7AkHACeBL4CljzF8icqeInAUwxiQYY04k3oBIIMEYc9IkH5xxQMeO\nHenXrx99+/alfv36/Pbbb5luvU2YMIFbbrmF1q1bc++993LHHXdQu3btFONL06dPp0uXLgwYMIAa\nNWrQtm1b1q5dS3nXLssVK1bk5ZdfZsCAAZQsWZKBAwfmxcdUDjLGtphOnoQvvrBbxql0XL4MffpA\n795OR6Jyke7F5wEuXrxI2bJlGTFiBP/5z3+cDkd5iJEjbdmMZcsg2TwbldqpU/Dww/YiffklBF8x\n10o5RPfi80Lr16/n66+/Zu/evfzxxx907dqVuLg4Onbs6HRoykNMnQrTpsEPP2hyytCWLXZK4+23\n210iNDn5FJ136QBjDG+//Ta7du0iMDCQ+vXrs3LlSkK1ToICvv0WXn8dli+HUqWcjsaD7d8PLVrA\n2LE6U89HaRefUh5k+XLo2NG2nBo2dDoaL7BvH1Ss6HQUKh057eLTBKWUh9i8Ge691w6j3HOP09Eo\nlXM6BuVHevToQRst+uOTduyABx6AiRM1OSmVSFtQXuTcuXMYYyhSxG7Ccdddd1GnTh3Gjx/vcGQq\nJ/btg2bNYMQI6N7d6Wg81OrVdgLEjTc6HYnKAm1B+YjY2NirHhMcHJyUnJRvOHLEtpheekmTU7o+\n/NBuQnj0qNORKDfTBJWO5cuX07hxY4KDgylWrBiNGzdm+/bthIeHExwczPfff0/16tUpVKgQLVq0\nSLHN0d69e2nXrh2lSpWicOHCNGzYMGkHiEQVK1Zk2LBh9OrVK2l3CYDhw4cTFhZGwYIFKVWqFE8+\n+WTSa5IvtrgsAAAe70lEQVR38fXo0YNly5YxceLEpEq/ERERVK1alTFjxqR4r127dhEQEJDm3n7K\nOSdP2uT0r39B375OR+OBLl60F2fcOFi5Elq2dDoi5WaaoNIQHx9Pu3btaNq0KVu2bGHdunU899xz\n5HMt5b906RLDhw8nPDycNWvWEB8fT4cOHZJef/78eVq1asWiRYv4888/6dixIw8//DA7d+5M8T5j\nx46lZs2a/P7774waNYo5c+YwevRopkyZwu7du1mwYAG3pLNt9bhx42jcuDE9evTg+PHjHD16lPLl\ny9OrVy+mTZuW4thp06ZRv359brrpply+Uiq7zpyxhV3bt4dBg5yOxgMdOmT7Pc+cgTVroFo1pyNS\nTshJrQ5Pu5FL9aAiIyNNQECAWb58+RXPffrppyYgIMCsXr066bH9+/ebfPnymUWLFqV7zttuu82M\nHDky6X5YWJhp06ZNimPGjBljatSoYeLi4tI8x5NPPmkeeuihpPvNmzc3/fr1S3HMsWPHTGBgoFnr\nKhgUHx9vypQpYyZNmpTBJ1budP68MbffbsyzzxqTkOB0NB5q+nRj3npLL5CXI4f1oLQFlYaQkBC6\nd+/Ovffey4MPPsjYsWM5dOhQ0vMBAQEp6jmVL1+e0qVLs337dgCio6N58cUXqV27NsWLFyc4OJjf\nf/+dAwdSbr6euHt5okceeYSYmBjCwsLo3bs333zzDZcvX85S7DfccAOtW7dOakX98MMPREZG0qVL\nlyydR+WN6Gh46CGoXt2uL9XNX9Px2GN2YE4vkF/TBJWOadOmsW7dOpo1a8a8efOoXr06v/zyS6Ze\nO2DAAGbPns3IkSNZvnw5mzdvplGjRlckm9S1ncqWLcvOnTv58MMPKVq0KAMHDqRhw4bExMRkKfbe\nvXszc+ZMLl68yCeffEKHDh1SFE1UzkhMTmXK2K2MAvS3T6kM6a9IBurUqcMLL7zAkiVLaNasGeHh\n4YAtKLh+/fqk4w4cOMCRI0eoVasWYGtFPfHEE7Rr144bb7yR0qVLs2fPnjTfI7XAwEAeeOABRo8e\nzbp169i2bRurVq1K99i06kHdf//9FClShMmTJzN//nx69eqV1Y+ucllMjJ2IVrIkfPqp7kyegpaK\nUenQBJWGiIgIBg8ezOrVqzlw4ABLlizhzz//TEpA+fLl4/nnn2fNmjVs2rSJ7t27U6dOHVq0aAHY\nWlFz585l48aNbNmyhW7dunHp0qWrvm94eDgff/wxW7duJSIigmnTphEYGEjVqlXTPD4sLIx169ax\nf/9+/vnnn6RqvQEBAfTo0YPBgwdTtmxZ7rrrrly6Mio7EpNTaCiEh2tySuHrr6FGDVsyWKlUNEGl\nISgoiJ07d/Loo49SvXp1evToQbdu3XjppZcAKFiwIC+//DJPPPEEjRs3RkSYPXt20uvHjBlDiRIl\naNq0Ka1bt6Zx48Y0adIkxXukVfK9WLFifPzxxzRt2pQ6deowd+5c5s6dS4UKFdKMc+DAgQQGBlKr\nVi1KlCjBwYP/q+HYs2dPLl++TM+ePXPjkqhsunjRztQrXhw++wyu0e2ZrUuXoF8/GDzY7o6rXdAq\nDbqTRBaFh4fTr18/zp49m6fvk1Nr166lSZMm7N27l7Jlyzodjl9KTE7BwTB9uianJPv3wyOP2MG4\nTz6BYsWcjkjlEd1JQqVw+fJlDh06xGuvvUaHDh00OTnk4kVbQ69wYbv5qyYnl8uXbYmMzp1hzhxN\nTipDmqB8zIwZMwgLCyMyMpLRo0c7HY5fio6GNm3g2mttyyl/fqcj8iCBgfD779C/v04hV1elXXxK\n5aJz5+DBB6FCBVsRV1tOyp9pF58HGzp0qNMhKDc6c8ZuF1ejhp1KrslJqZzxuQSVxY0XlMoVp07Z\noZXbboMpU3QRLhcvwjPPwOefOx2J8mI+18UXGWkICXE6EuVPjh2zu5K3aQMjR+rQCn/9BZ062abk\nhx/qRAg/pl18qVy44HQEyp8cPAhNm9pJaaNG+XlyMgY++giaNLFrnGbO1OSkckQTVB7SMSjftmeP\nrQjRpw+88orT0XiAV1+F8eNh+XJbx8mvs7XKDZqglMqGTZtsy+nFF2HAAKej8RB9+sDateDaEkyp\nnPK5MahlywxNmzodifJly5bZjRAmTYKOHZ2ORinPldMxKJ+bCKstKJWXvvvO9l7NmAF33+10NEr5\nNu3iy0M6BuVbpk2Dp56ChQv9ODklJNhxpr59nY5E+QFtQSl1FcbAu+/C5Mm2e69aNacjcsihQ9Cj\nB5w/r+ublFtoCyoPaQvK+yUkwAsv2FIZK1f6cXKaORMaNrTTFlesgCpVnI5I+QGfa0GdP+90BMpX\nXLoEvXrB3r125nTx4k5H5JDwcHjzTViwAG6+2elolB/RFlQe0haU9zp9Gu67z1bDXbTIj5MT2CmL\nf/yhyUm5nSYopVLZtw9uvx0aNLAVyQsVcjoihwUF2ZtSbqYJKg9pC8r7rF8Pd9wBTz8NY8ZAvnxO\nR+Rmp087HYFSSTRBKeUybx60amVn6/Xr53Q0bnbypN1QsFs3pyNRKonPJShPmiShLSjv8f77do3T\nggXQtq3T0bjZN99A3bpQtizMmuV0NEol8blZfGfOOB2B8iZxcTBwIPz4I6xaBRUrOh2RGx05As8+\nC1u3wuzZduBNKQ/ic3vx1a1r2LzZ6UiUNzhzxvZqxcXZyRB+N1Nv7lw7O+/ll6FgQaejUT4op3vx\n+VyCKl3acPiw05EoT7drly0weM89djJE/vxOR6SU7/HagoUiEiIic0XkvIjsE5HH0jluoIhsEZGz\nIrJHRAZmdN5Tp+zWNJ5Ax6A806JFcOed8NxzMGGCJielPJWTkyQmAReBUOBxYLKI1Ezn2G5AMeAB\n4BkReTS9k15zjc7kU+mbNAm6doWvvrKTIvzC4sW20q1SXsaRLj4RCQJOA7WMMXtcj30GHDLGDLnK\na8cBGGOeS+M5U66cYflyCAvL/biV94qNtS2mpUth/nyoXNnpiNzg5Em7keDixXbufOvWTkek/Iy3\ndvFVA+ISk5PLZqB2Jl7bBNiW3pPXXw///JPD6JRPOXHCblu0fz+sXu0HySk+HiZOhNq17cyPbds0\nOSmv5NQ088JAVKrHooDgjF4kIsMAAT5J75gzZ4YydqzdbLl58+Y0b948p7Fm29ChQ3UcymHr1tmq\nt926wfDhfrIzRP/+sHmzbTndeKPT0Sg/snTpUpYuXZpr53MqQZ0HiqR6rAhwLr0XiMgz2LGqO40x\nsekdd+utQ2nVCrp0yZU4lRebOtXOoJ461c8W344YAcHBINnuWVEqW1I3CoYNG5aj8zmVoHYC14hI\n5WTdfPVIp+tORHoCLwJNjDFHMzpxaCgcP56rsWabtp6ccfGi3arot99s6aLq1Z2OyM2KpP7bTynv\n5MgYlDEmGpgDDBeRIBG5A2gDXFGmU0S6AiOBlsaY/Vc7d9my6DooP3bwIDRtahfhrlnj48lp1So7\nsKaUj3JymnlfIAg4AXwJPGWM+UtE7hSRs8mOGwEUB9aLyDnXeqhJ6Z20bFlbmdoTaAvKvRYvhltu\ngUcftTtDBGc4ounFIiKgUye7DYYmKOXDHNuLzxhzGmifxuMrSTY+ZYyplJXzelKCUu4RHw8jR9qZ\n1F9+CS1aOB1RHjl71la2/fBDO2f+k0+0TpPyaT63WawnJShtQeW9Y8fswtuEBPj9dyhd2umI8sil\nS1CvHjRrBn/+CWXKOB2RUnnO5/bii4kxFCliB8oDfK6YiEpu0SJ44gno3Rtee80PppAfOwYlSzod\nhVKZ5q0LdfNMwYJQrJhnzOTTFlTeiI+H11+3a5vCw2HYMD9ITqDJSfkdn0tQYGv67N3rdBQqLxw9\nancgX7nSVoq45x6nI8plBw7Y7dWVUr6ZoKpWhd27nY5CW1C5bcECaNAAmjeHn3/2sQbFP//Yyon1\n60NkpG0mKuXnfG6SBNgEtWuX01Go3BIdbfc8XbAAZs6065x8RnQ0jBtnW00dO9rqtqVKOR2VUh7B\nZ1tQnpCgtAWVcxs3QsOGduHtpk0+lpzA1v/YuNEuup08WZOTUsloC0p5pIQE+L//g3ffhffes1PJ\nfdKAAbpnnlLp8Llp5sYYzp+HG26AqChbwFB5l4MH7fTx+Hj47DMfqe0VF2enGmoyUn5Ep5mnoXBh\n21PiCRMlVOYZA198Ybv0WraEJUt8IDnFx9sPVbMmLF/udDRKeRWfTFAAdevaBfdO0jGozDt6FNq1\ng3fegR9/hCFDvHxtU1ycTUy1a8MHH9jtiZo1czoqpbyKTyeoTZucjkJdTWKrqV49+zPbsMFOJfdq\nO3bYgdCPPoIJE2zL6a67nI5KKa/jk2NQAAsX2kH2xYsdDkql6+hReOopu6j6009t155PuHzZZtrb\nb3c6EqUcpWNQ6bj9dli/HmLTrb2rnGKM3XX8ppugTh37Xe4zyQkgMFCTk1K5wGcTVLFiUKmSXWLi\nFB2DutL+/fDQQ/DWW3bh7RtvQIECTkeVDRER8N//2mmGSqk84bMJCuDOO23Jb+W8uDgYPdq2lBo3\ntqUxbr7Z6aiyYe1aWxHx5pttS8lni08p5TyfHYMCmDHDVladO9fBoBQbNsC//w0hITBlip0/4HVO\nnrTTDI8eheefhx49fLhkr1K5I6djUD6doA4dsntvHj+utaGccO4cvPoqfPWVnT7erZsXr1M1xs68\nuf9+L5//rpT76CSJDJQtC6GhdrKEE/x1DMoYu6lr7dp2D72tW+3OEF6TnBISrnxMBFq31uSklBv5\ndIICaN9eu/jcaetWOywzapRd3/Tpp3D99U5HlQmxsfDNN3a90v/9n9PRKKXw8S4+sOMfXbvatZNe\n8xe8F4qKgqFD7fTx116z65u8Yh/Eo0ftLg8ffgiVK0PfvvavmsBApyNTyutpF99VNGwIMTHw119O\nR+KbEhJs2fUaNeD8edi2DZ55xkuS04EDUKuWTVI//mh3fOjUSZOTUh7C5xOUCHToYAfq3c3Xx6CW\nLoVbbrEljb77DqZOtWN+XqN8eTuTZsoUu2JYKeVRfD5BAfTuDR9/rLtK5JYdO6BtWzvTeuBAWL3a\nJiqPFBNjB8PSKxB27bXujUcplWl+kaBuvNHuKvH99+59X19rQZ04YYdomjSxt7/+gs6dPXAKf0KC\nrdXRqxeUKWMHxs6fdzoqpVQWedpXS5556inbFaWy7vx5GDnSDtfkz29bUAMHQsGCTkeWhkWLbNdd\n//424C1b4Icf7II4pZRX8ZsE1bEj/P03rFvnvvf09hZUTIzdnqhyZTv5Yc0aW379uuucjiwDN95o\nJzxs3GjLqZcp43RESqls8psEVaAADB5sp0KrjF26BBMnQpUqsGoV/PorTJ9u73uEvXvh/fftiuDU\nbrjBJimllNfz+XVQyV26BNWq2S/bO+5wY2BeIjbWThkfMcJ+xw8f7kFlMHbtsgtpv/kGDh60a5VG\nj4bChZ2OTCmVDt2LL5mrJSiwG8i+/bZdwOsVa3XcIDraznJ8912bwIcP97ByRk88Ab/8YtcLPPKI\n3aZef3hKeTxNUMlkJkEZAy1bQqtWdhw9Lw0dOtSjx6GiouzEkXHjbAmMwYM9dLr40aNQooTug6eU\nl9GdJLJIBCZPhjfftBO8/NHhwzYZVa4M27fbiW9z5zqUnPbvtz+Q1q1h0KC0jylVSpOTUn7I71pQ\nicLDbVffunX+M4yxdq2dhffTT3Z/wv79oWJFBwKJiLD1N375Bc6ehXvvtWV277sPihZ1ICClVF7Q\nLr5kspKgjIF//cvWivr2W9/9Az02FmbPtonp+HF49lno2dPhPHDkiB0MbNnSzsbwuJW+SqncoAkq\nmawkKLBf3q1aQVgYfPBB7n9POjkGtXu3nfjw6adQvbotAvvQQ25IxFFR8NtvsGKFreu+cKHvZn+l\nVIZ0DCoH8ueHOXPsAt5evbx/r76LF+0U+hYt7Cy82FhYvNhu6tquXR7niUGD7G4NZcrYvtN8+exC\nWR/6A0gp5V5+3YJKdOECPPqonW799dfetSN3QoJtrMyYYZcINWxoN8dt2zYPqkZERtr9jYKCrnxu\n+nTbFG3Y0K6KVkr5Pe3iSya7CQogPh5efdV+z37yiS2s6qmMseu4ZsywpdWvvx4ee8zeKlTIpTc5\netRuF/Tnn7B5M6xfb3eLnTcPmjfPpTdRSvkyr+3iE5EQEZkrIudFZJ+IPJbBsW+LyCkROSkib+dF\nPPny2TLl778P3bvbiQQnTuTsnLk5/nTxot1irm9fm4S6dLGVIn75xeaPQYOymZzi49N+fPx4O7Pi\n1Cl44AGYPx/OnNHkpJRyGyeX408CLgKhQANggYhsMsakqH0rIn2ANkBiRblfRWSPMebDvAjqwQeh\nWTNbtrxmTTs29d//2qU47mSMHRtbutROC1+82NbUe+ghm6hq1sxaCfulX39N84AA2Lkz5W3AALso\nKrU338y1z+INli5dSnNNvmnSa5MxvT55x5EWlIgEAR2AV4wxMcaYVcA8oFsahz8BjDbGHDXGHAVG\nA0+me/Lrr7fjIB062Klr2SilGxwMY8falkl0tK3a0K6dna597lzmz5OVFlRMjN0tfOJE21VXurRd\nFrRmjf0oe/bAypXw0ks2nqTkFBcHx47ZPr85c+zurmlY+tVXti5SVJQt5vT223aL8vQWx/qZpUuX\nOh2Cx9JrkzG9PnnHqRZUNSDOGLMn2WObgaZpHFvb9Vzy42qne+bt2+3uBIm36Oi0j1u+HIYNs7tf\nlyxpt9IJCbFNk6Y2jLJlbZffm2/ayRMffghPPgmNGtldFxo1grp1oVy5zNVGMsbmh/377d6nO3fa\nVtLGjbB7l+GmatHcXjuKjrWjGP1IFKWvjbK1LW6++cqTzZoFTz9tu91CQmywFSrYbJbWTrh16+pW\n7kopr+JUgioMRKV6LAoIzsSxUa7H0laihL01apRxBLVr29bD8eP/u+3da7c8b5oyTwYHQ68is+i1\nuhcJBQtyaWtBojcX4NykgvwQ8CDPx7xJ8eL2bQsXtrd6kUtos/VVChQoRHwCxF+ORy5fYlX+uwiv\n+gZVq0LVqnbf0379oN62GeT/T284URT+KGpX0hYtasd/0kpQ990HW7faFqOuM1JK+SBHZvGJyE3A\nSmNM4WSP9QeaGWPapjr2DHCPMWaD634DYIkx5oq9EETEd6YkKqWUD8jJLD6nWlA7gWtEpHKybr56\nwLY0jt3mem6D6/5N6RyXowuhlFLKszgyScIYEw3MAYaLSJCI3IGdqfd5God/BvQXkdIiUhroD3zi\nvmiVUko5wcmtjvoCQcAJ4EvgKWPMXyJyp4icTTzIGPMBMB/YAvwJzDfGTHUiYKWUUu7jUztJKKWU\n8h1+vVmsUkopz+VVCcrTtkfyNJm9PiIyUES2iMhZEdkjIgPdHau7ZeXfjuv4/CKyQ0QOuCtGJ2Xx\nd6uBiCwTkXMiclRE+rkzVnfLwu9VoIhMEZFjru+e70TEzXvQuJeI9BWR9SJyUUSmXeXY/7r+vZwW\nkY9EJP/Vzu9VCYqU2yM9DkwWkZqpD0q1PVJd4EER+bc7A3VIpq6PSzegGPAA8IyIPOqeEB2TlWsD\n8CJwzB2BeYjM/m5dB/wATAZCgCrAz26M0wmZ/bfzPHArcCNQGrtmc4K7gnTIYWAE8HFGB4nIfdjf\nqbuAMKAyMOyqZzfGeMUNO6HiElA52WOfAaPSOHYV0DvZ/Z7Ab05/Bk+5Pmm8dhwwzunP4CnXBqiI\nXcpwH3DA6fg96foAI4Fwp2P20GszCXgr2f1WwF9OfwY3XacRwLQMnv8SeCPZ/RbA0aud15taUOlt\nj5TWtkdZ2x7JN2Tl+qTWhHTWlvmIrF6b8cBg7F/N/iAr1+c24LSIrBKR465urHJuidIZWbk2HwN3\nikgp136jXYGFbojRG6T1nVxCREIyepE3Jai82x7JN2Tl+iQRkWGA4NtryzJ9bUSkPZDPGDPPHYF5\niKz82ymL3cC5H1AOiABm5GVwDsvKtdkJHMB2e50BamBbFirt72ThKt9P3pSgzgNFUj1WBEhrf/HU\nxxZxPebLsnJ9ABCRZ7B96q2MMV5e8D5Dmbo2rr9638Z++YL9BfIHWfm3EwPMNcb8YYy5jB1HuF1E\nMvyi8WJZuTZTgALYsblrgbnAj3kanfdI6zvZkMH3E3hXgkraHinZY1fbHilRutsj+ZCsXB9EpCd2\n0LKFsWVMfFlmr01VoAKwQkSOArOB0iJyRETKuydUR2Tl386f2C+W5Ay+m8yzcm3qAp8aY6Jcf/BN\nAG4RkeJuiNPTpfWdfNwYczrDVzk9uJbFgbjp2MG2IOAO4DRQM43j+rguSGnXbSvwL6fj96Dr0xU4\nClR3OmZPujbYP9hKJLu1Bw5hZ2+J05/B6evjOu4u4B/sl3F+YCywzOn4PeTaTANmYVsH+YEhwEGn\n48/ja5MPKAiMwk4eKYDtIk993H3AEaAmtoW5CBh51fM7/QGzeDFCsM3m89i+706ux+8EzqY69i3X\nL9Ip4E2nY/ek6wPsxc5MOottYp8FJjkdvydcm1SvaYYfzOLL6vXB/gF4yPX79R1Qxun4PeHaAMWB\nL4DjQCSwHLjZ6fjz+Nq8DiQA8clur2HHJ88BZZMd+zx26cYZ4CMg/9XOr1sdKaWU8kjeNAallFLK\nj2iCUkop5ZE0QSmllPJImqCUUkp5JE1QSimlPJImKKWUUh5JE5RSSimPpAlKKTcQkWYikuCN296I\nyBIRGe90HMr/aIJSPkVEbhKROBFZkY3Xvi4iW/IiLhdvXRXfHlt+BABXVdn+Dsaj/IQmKOVr/gVM\nBG4UkerZeL23JpFsyUzZbWPMGWPMBXfEo1RymqCUzxCRgkAXYCrwDdA7jWNKiciXInJKRC6IyB+u\n7rfu2H3Faru64uJF5AnXaxJEpEOq86RoRYjIf0Vks4icF5FDIjJVRIpmMf4OrnNEi8g/rq61UNdz\nr4vIFhHpJSL7XcfMdZVgT3z9zSLyk4icFJEoEVkhIreleo8EEXlaRGaLyHlgpIhcIyLjReSwiFx0\nnX9UstckdfGJyBLsju/vJrtOQa73S32NWorI5cTPoFRWaYJSvuQRIMIYsxW7aecTIpIv8UlXvafl\nQHmgLXAjMNz19FfAaOBv4AagFDAzC+8dDzwH1AIeAxphK/NmiojcgC389wm20F0T4PNUh4Vhd6J/\nCLgbWx7k42TPB2N3lL7D9f4bgQVpjHu9BizAfv6JwLPY6/EoUAXohL0OaemA3Sh2GFASKGWMiXbF\n3jPVsT2AecaYkxl+eKXScY3TASiVi3phv6AxxiwTkQtAG+xO1GC/3EsAt5j/1aHZl/hiV4siLjtf\nqMaY5MnogIi8BHwLdM/kKUpjfx9nG2MOuh7bnuqYgkA3Y8xhV7x9sLWrKhtj9hhjliQ/WESeAzoC\n92NLRiT6yhgzLdlxFYCdxphVrocOAWvS+ZynRSQeOG+MOZHsqanAahEpZYw5KiLFgHbAw5n8/Epd\nQVtQyieISBVsyyF5+fHp2DGpRDcBf5qrFUnL3vu3EJGfReSgiJwF5gCBIlIyk6fYjK2Rs01EvhGR\np0Tk+lTHHE5MTi5rsaUOarpiCBWRD0TkbxE5gy2jEoptMSb3e6r7nwL1RWSniLwvIq1EJEsFCI0x\nv2PrriUm5K7YkhNaUVZlmyYo5St6Y/89HxSRWBGJBV4CWopIGdcx2a36mlbF2KTJBa5qu99ji2R2\nBBrwv+6uwEy9gTEJxph7gZbYZNUL2CUidbIQ52dAQ2xXY2NsBdPDacSQYsKDMWYjdlxpMPZzhgM/\nZ+F9E32E7dbD9d9PjNbzUTmgCUp5Pdc40xPAIOyXcvLbn/zvS/MPoG4Ga5EuYyuEpnYSOyaV+H43\nJL8P3IxNWP2NMWuNMbuBMmSD6/UjjDGNsBVIOyV7ukyyZAtwKzahJHYF3gFMMMb8aIz5C5uIkseZ\n0fteMMbMNsb0BVoDd7tapWlJ7zp94YqxL1Af2zJTKts0QSlf8CBwHfCRMWZ78ht2okMv13HTgRPA\ntyJyp4iEichDItLM9XwEUEFE6ovIdSKS2PJYDPQVkYYiUh87kSEm2fvvwv4u/dd1zsewrZjU0m3B\nicitIvKyayZeORFpC5TFtsoSXQTCRaSeiDQGJgPfG2P2up7fCTwuIjVFpBG2u/PSVa5d4gzEziJS\nw5WUugJR2LGotEQATUSkdPJZhMaYs9jZk6OxZeD3XO29lcqIJijlC3oCi9MZW5oFlBeRe1yzzZph\nu73mYcdMhvK/tU+zgYXYsaATQGfX4wOAvcAS4GvshICkCQLGmC3YhPRfbELp6XpNahl1d0VhW0Dz\nsYnmXWC4MSb5mNo+7GzD+cCvwG5SzpzrARQGNmCT8cfYZHK1GM4BL2DHtDYAdYEHjDEX03lNYknv\nPSS7Di4fY7sUP0apHNKS70p5ARF5HXjYGFPX6VgyIiKdsC270skSnFLZotPMlVI5JiKFsONdg4EP\nNTmp3KBdfEqp3PAisAM4BbzhcCzKR2gXn1JKKY+kLSillFIeSROUUkopj6QJSimllEfSBKWUUsoj\naYJSSinlkf4fSRC3hLflMqQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f883c724e80>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "p = 0.1\n",
    "q = np.linspace(0, 1, 500)\n",
    "kl_div = p * np.log(p / q) + (1 - p) * np.log((1 - p) / (1 - q))\n",
    "mse = (p - q)**2\n",
    "plt.plot([p, p], [0, 0.3], \"k:\")\n",
    "plt.text(0.05, 0.32, \"Target\\nsparsity\", fontsize=14)\n",
    "plt.plot(q, kl_div, \"b-\", label=\"KL divergence\")\n",
    "plt.plot(q, mse, \"r--\", label=\"MSE\")\n",
    "plt.legend(loc=\"upper left\")\n",
    "plt.xlabel(\"Actual sparsity\")\n",
    "plt.ylabel(\"Cost\", rotation=0)\n",
    "plt.axis([0, 1, 0, 0.95])\n",
    "save_fig(\"sparsity_loss_plot\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def kl_divergence(p, q):\n",
    "    \"\"\"Kullback Leibler divergence\"\"\"\n",
    "    return p * tf.log(p / q) + (1 - p) * tf.log((1 - p) / (1 - q))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 1000  # sparse codings\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01\n",
    "sparsity_target = 0.1\n",
    "sparsity_weight = 0.2\n",
    "\n",
    "#activation = tf.nn.softplus # soft variant of ReLU\n",
    "activation = tf.nn.sigmoid\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "\n",
    "weights1_init = initializer([n_inputs, n_hidden1])\n",
    "weights2_init = initializer([n_hidden1, n_outputs])\n",
    "\n",
    "weights1 = tf.Variable(weights1_init, dtype=tf.float32, name=\"weights1\")\n",
    "weights2 = tf.Variable(weights2_init, dtype=tf.float32, name=\"weights2\")\n",
    "\n",
    "biases1 = tf.Variable(tf.zeros(n_hidden1), name=\"biases1\")\n",
    "biases2 = tf.Variable(tf.zeros(n_outputs), name=\"biases2\")\n",
    "\n",
    "hidden1 = activation(tf.matmul(X, weights1) + biases1)\n",
    "outputs = tf.matmul(hidden1, weights2) + biases2\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "mse = tf.reduce_mean(tf.square(outputs - X))\n",
    "\n",
    "hidden1_mean = tf.reduce_mean(hidden1, axis=0) # batch mean\n",
    "sparsity_loss = tf.reduce_sum(kl_divergence(sparsity_target, hidden1_mean))\n",
    "loss = mse + sparsity_weight * sparsity_loss\n",
    "training_op = optimizer.minimize(loss)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train MSE: 0.0808103 \tSparsity loss: 0.415093 \tTotal loss: 0.163829\n",
      "1 Train MSE: 0.0544481 \tSparsity loss: 0.118671 \tTotal loss: 0.0781823\n",
      "2 Train MSE: 0.0489213 \tSparsity loss: 0.104058 \tTotal loss: 0.0697329\n",
      "3 Train MSE: 0.042664 \tSparsity loss: 0.0575809 \tTotal loss: 0.0541802\n",
      "4 Train MSE: 0.0390694 \tSparsity loss: 0.259185 \tTotal loss: 0.0909064\n",
      "5 Train MSE: 0.0361558 \tSparsity loss: 0.0221424 \tTotal loss: 0.0405843\n",
      "6 Train MSE: 0.0328707 \tSparsity loss: 0.0500909 \tTotal loss: 0.0428889\n",
      "7 Train MSE: 0.0294563 \tSparsity loss: 0.0537512 \tTotal loss: 0.0402066\n",
      "8 Train MSE: 0.027672 \tSparsity loss: 0.0859079 \tTotal loss: 0.0448536\n",
      "9 Train MSE: 0.0257369 \tSparsity loss: 0.0981425 \tTotal loss: 0.0453654\n",
      "10 Train MSE: 0.0230508 \tSparsity loss: 0.142276 \tTotal loss: 0.0515059\n",
      "11 Train MSE: 0.0230676 \tSparsity loss: 0.181762 \tTotal loss: 0.05942\n",
      "12 Train MSE: 0.020677 \tSparsity loss: 0.0929343 \tTotal loss: 0.0392639\n",
      "13 Train MSE: 0.0190676 \tSparsity loss: 0.0617354 \tTotal loss: 0.0314146\n",
      "14 Train MSE: 0.0211431 \tSparsity loss: 0.0756601 \tTotal loss: 0.0362751\n",
      "15 Train MSE: 0.0172402 \tSparsity loss: 0.106335 \tTotal loss: 0.0385071\n",
      "16 Train MSE: 0.0173277 \tSparsity loss: 0.0966078 \tTotal loss: 0.0366493\n",
      "17 Train MSE: 0.0160216 \tSparsity loss: 0.0766543 \tTotal loss: 0.0313525\n",
      "18 Train MSE: 0.0152365 \tSparsity loss: 0.0632296 \tTotal loss: 0.0278824\n",
      "19 Train MSE: 0.0190664 \tSparsity loss: 0.383607 \tTotal loss: 0.0957877\n",
      "20 Train MSE: 0.0150545 \tSparsity loss: 0.0546298 \tTotal loss: 0.0259805\n",
      "21 Train MSE: 0.0148168 \tSparsity loss: 0.0451466 \tTotal loss: 0.0238461\n",
      "22 Train MSE: 0.0146978 \tSparsity loss: 0.227617 \tTotal loss: 0.0602212\n",
      "23 Train MSE: 0.0150732 \tSparsity loss: 0.135743 \tTotal loss: 0.0422218\n",
      "24 Train MSE: 0.0177874 \tSparsity loss: 0.194969 \tTotal loss: 0.0567811\n",
      "25 Train MSE: 0.0142951 \tSparsity loss: 0.10445 \tTotal loss: 0.035185\n",
      "26 Train MSE: 0.0142418 \tSparsity loss: 0.0933622 \tTotal loss: 0.0329143\n",
      "27 Train MSE: 0.0149857 \tSparsity loss: 0.100697 \tTotal loss: 0.0351252\n",
      "28 Train MSE: 0.0161774 \tSparsity loss: 0.0489748 \tTotal loss: 0.0259723\n",
      "29 Train MSE: 0.0158215 \tSparsity loss: 0.375802 \tTotal loss: 0.0909819\n",
      "30 Train MSE: 0.0159297 \tSparsity loss: 0.0585507 \tTotal loss: 0.0276398\n",
      "31 Train MSE: 0.0131048 \tSparsity loss: 0.0708771 \tTotal loss: 0.0272802\n",
      "32 Train MSE: 0.0133353 \tSparsity loss: 0.274087 \tTotal loss: 0.0681527\n",
      "33 Train MSE: 0.0134352 \tSparsity loss: 0.089336 \tTotal loss: 0.0313024\n",
      "34 Train MSE: 0.011721 \tSparsity loss: 0.0931978 \tTotal loss: 0.0303606\n",
      "35 Train MSE: 0.0134944 \tSparsity loss: 0.121721 \tTotal loss: 0.0378386\n",
      "36 Train MSE: 0.0130126 \tSparsity loss: 0.0709383 \tTotal loss: 0.0272003\n",
      "37 Train MSE: 0.0137884 \tSparsity loss: 0.0827571 \tTotal loss: 0.0303398\n",
      "38 Train MSE: 0.0138806 \tSparsity loss: 0.192237 \tTotal loss: 0.0523281\n",
      "39 Train MSE: 0.012782 \tSparsity loss: 0.162988 \tTotal loss: 0.0453796\n",
      "40 Train MSE: 0.0129845 \tSparsity loss: 0.184135 \tTotal loss: 0.0498114\n",
      "41 Train MSE: 0.0141138 \tSparsity loss: 0.145071 \tTotal loss: 0.043128\n",
      "42 Train MSE: 0.0137911 \tSparsity loss: 0.106032 \tTotal loss: 0.0349975\n",
      "43 Train MSE: 0.0121966 \tSparsity loss: 0.129571 \tTotal loss: 0.0381109\n",
      "44 Train MSE: 0.0146962 \tSparsity loss: 0.406566 \tTotal loss: 0.0960094\n",
      "45 Train MSE: 0.0124659 \tSparsity loss: 0.202119 \tTotal loss: 0.0528897\n",
      "46 Train MSE: 0.0119216 \tSparsity loss: 0.0971551 \tTotal loss: 0.0313527\n",
      "47 Train MSE: 0.0107826 \tSparsity loss: 0.132033 \tTotal loss: 0.0371893\n",
      "48 Train MSE: 0.0119806 \tSparsity loss: 0.118638 \tTotal loss: 0.0357083\n",
      "49 Train MSE: 0.0113738 \tSparsity loss: 0.317469 \tTotal loss: 0.0748676\n",
      "50 Train MSE: 0.0126131 \tSparsity loss: 0.130214 \tTotal loss: 0.038656\n",
      "51 Train MSE: 0.0115075 \tSparsity loss: 0.126815 \tTotal loss: 0.0368705\n",
      "52 Train MSE: 0.0131578 \tSparsity loss: 0.239686 \tTotal loss: 0.061095\n",
      "53 Train MSE: 0.011203 \tSparsity loss: 0.144113 \tTotal loss: 0.0400257\n",
      "54 Train MSE: 0.0126501 \tSparsity loss: 0.373799 \tTotal loss: 0.0874099\n",
      "55 Train MSE: 0.0119187 \tSparsity loss: 0.371901 \tTotal loss: 0.0862989\n",
      "56 Train MSE: 0.0112231 \tSparsity loss: 0.225509 \tTotal loss: 0.0563249\n",
      "57 Train MSE: 0.0113873 \tSparsity loss: 0.107863 \tTotal loss: 0.0329598\n",
      "58 Train MSE: 0.0108124 \tSparsity loss: 0.0912845 \tTotal loss: 0.0290693\n",
      "59 Train MSE: 0.0117589 \tSparsity loss: 0.257519 \tTotal loss: 0.0632627\n",
      "60 Train MSE: 0.0111196 \tSparsity loss: 0.232198 \tTotal loss: 0.0575592\n",
      "61 Train MSE: 0.0114572 \tSparsity loss: 0.138591 \tTotal loss: 0.0391755\n",
      "62 Train MSE: 0.0117489 \tSparsity loss: 0.116193 \tTotal loss: 0.0349875\n",
      "63 Train MSE: 0.0111976 \tSparsity loss: 0.262772 \tTotal loss: 0.063752\n",
      "64 Train MSE: 0.0117205 \tSparsity loss: 0.145903 \tTotal loss: 0.0409011\n",
      "65 Train MSE: 0.0166962 \tSparsity loss: 1.01099 \tTotal loss: 0.218893\n",
      "66 Train MSE: 0.0207524 \tSparsity loss: 0.392595 \tTotal loss: 0.0992715\n",
      "67 Train MSE: 0.0156945 \tSparsity loss: 0.187199 \tTotal loss: 0.0531343\n",
      "68 Train MSE: 0.0194048 \tSparsity loss: 0.515735 \tTotal loss: 0.122552\n",
      "69 Train MSE: 0.0116519 \tSparsity loss: 0.835861 \tTotal loss: 0.178824\n",
      "70 Train MSE: 0.0209426 \tSparsity loss: 0.292206 \tTotal loss: 0.0793837\n",
      "71 Train MSE: 0.0134064 \tSparsity loss: 0.542741 \tTotal loss: 0.121955\n",
      "72 Train MSE: 0.0151789 \tSparsity loss: 0.376449 \tTotal loss: 0.0904687\n",
      "73 Train MSE: 0.0152982 \tSparsity loss: 0.37529 \tTotal loss: 0.0903562\n",
      "74 Train MSE: 0.023566 \tSparsity loss: 0.454576 \tTotal loss: 0.114481\n",
      "75 Train MSE: 0.0164631 \tSparsity loss: 0.456268 \tTotal loss: 0.107717\n",
      "76 Train MSE: 0.0164126 \tSparsity loss: 0.214973 \tTotal loss: 0.0594073\n",
      "77 Train MSE: 0.0159823 \tSparsity loss: 0.330593 \tTotal loss: 0.0821008\n",
      "78 Train MSE: 0.0297457 \tSparsity loss: 0.148935 \tTotal loss: 0.0595327\n",
      "79 Train MSE: 0.017431 \tSparsity loss: 0.273009 \tTotal loss: 0.0720328\n",
      "80 Train MSE: 0.0280243 \tSparsity loss: 0.491862 \tTotal loss: 0.126397\n",
      "81 Train MSE: 0.0199488 \tSparsity loss: 0.279786 \tTotal loss: 0.075906\n",
      "82 Train MSE: 0.020573 \tSparsity loss: 0.319931 \tTotal loss: 0.0845592\n",
      "83 Train MSE: 0.0156649 \tSparsity loss: 0.153916 \tTotal loss: 0.0464482\n",
      "84 Train MSE: 0.0145016 \tSparsity loss: 0.989233 \tTotal loss: 0.212348\n",
      "85 Train MSE: 0.0287968 \tSparsity loss: 0.544559 \tTotal loss: 0.137709\n",
      "86 Train MSE: 0.0122126 \tSparsity loss: 0.0891289 \tTotal loss: 0.0300384\n",
      "87 Train MSE: 0.0145746 \tSparsity loss: 1.48072 \tTotal loss: 0.310719\n",
      "88 Train MSE: 0.0183811 \tSparsity loss: 1.096 \tTotal loss: 0.237581\n",
      "89 Train MSE: 0.0294373 \tSparsity loss: 1.21178 \tTotal loss: 0.271793\n",
      "90 Train MSE: 0.0211941 \tSparsity loss: 0.141069 \tTotal loss: 0.0494079\n",
      "91 Train MSE: 0.0131894 \tSparsity loss: 0.263666 \tTotal loss: 0.0659225\n",
      "92 Train MSE: 0.0178111 \tSparsity loss: 0.566228 \tTotal loss: 0.131057\n",
      "93 Train MSE: 0.0142184 \tSparsity loss: 0.368309 \tTotal loss: 0.0878801\n",
      "94 Train MSE: 0.0133273 \tSparsity loss: 0.133334 \tTotal loss: 0.039994\n",
      "95 Train MSE: 0.0148034 \tSparsity loss: 0.167435 \tTotal loss: 0.0482905\n",
      "96 Train MSE: 0.0207172 \tSparsity loss: 0.236838 \tTotal loss: 0.0680848\n",
      "97 Train MSE: 0.0379609 \tSparsity loss: 0.521251 \tTotal loss: 0.142211\n",
      "98 Train MSE: 0.0187403 \tSparsity loss: 0.370602 \tTotal loss: 0.0928608\n",
      "99 Train MSE: 0.0170631 \tSparsity loss: 1.34448 \tTotal loss: 0.285958\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 100\n",
    "batch_size = 1000\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = mnist.train.num_examples // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch})\n",
    "        mse_val, sparsity_loss_val, loss_val = sess.run([mse, sparsity_loss, loss], feed_dict={X: X_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_val, \"\\tSparsity loss:\", sparsity_loss_val, \"\\tTotal loss:\", loss_val)\n",
    "        saver.save(sess, \"./my_model_sparse.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa0AAAFxCAYAAADAqvdjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHVpJREFUeJzt3W9slXf5x/FvN0ppC6Wj9A8w2GDIQGKRkWVM558tCFEM\nbixLRB9o5hIJyZZIogskS9Qn7pnyxKjRAKIxaHQayP6kU0kmFEEcEx0if3WMlBZoaWlpy5/5yAe/\n3/W5zP2lp6de5f16eOV77nOf+3B6cef+5PpWvPfeewkAgAjuGOsTAACgKJoWACAMmhYAIAyaFgAg\nDJoWACAMmhYAIAyaFgAgDJoWACAMmhYAIIwJZXwvRm+gHCrG+gRuB11dXeb3fOPGDbn2jjvs/41v\n3rwp11ZU2K9P1bxjeGtV3TuHa9eumVplZaVcq9x5552F30tdG+86qs+QM9HIuzY51PvlfD/edbx+\n/bqptbS0yANzpwUACIOmBQAIg6YFAAijnM+0AIwTOc82FG/tSI+b837q2VMp3k89E5swQf+pzXkm\npdaqZ2Ip5X1e9TzJO6+c52o5zzK9zyHXFl4JAMAYo2kBAMKgaQEAwqBpAQDCoGkBAMIgPQggm0qB\neek0JSehV4okm5oy4SXZ1NQGb0rF8PBwoXNQCT3vHLykoap7a9V5qZp3jJxUo/e9e9d3pGu50wIA\nhEHTAgCEQdMCAIRB0wIAhEEQA0BJeKN4csYEqQfyOSN+crY8yRmtlBNMUEEO7/WTJk0qfF4qDJIz\nDsu7juq43jko3jXPGcnFGCcAwLhE0wIAhEHTAgCEQdMCAIRB0wIAhEF6EEC2oaEhU/OSYTkjm1SK\nzEuW5ST6Jk6cWPi8vJFLihphpNJ03nupterapqRTfmrDSe/9BgcH5Vp1fVUCMiX9eXOSoN44LNKD\nAIBxiaYFAAiDpgUACIOmBQAIgyAGgGxq/JD3QD5npFDOA3kVNqiuri681nsvNcLICxCo0IYKIHjB\nBnUO/f39cq13DooKnnjXRsnZGy0niJGzb5aHOy0AQBg0LQBAGDQtAEAYNC0AQBg0LQBAGKQHAWRT\nabycsUpeEk4d1xtVNDw8bGo5mxd651tTU2NqKi2Zkk7ZqUSgl8ZT6UPvvVQC0Tuuuo7qeqWU0pUr\nVwqdV0r6s+WkGnNSiR7utAAAYdC0AABh0LQAAGHQtAAAYdxWQYz9+/eb2pYtW+TaWbNmmZo3BuUL\nX/iCqU2bNk2u9epAJCMdx+OFINReUufPn5dr29vbTe3UqVNyrQoLNDQ0yLUzZswwtfnz58u1zc3N\npqaCHKrm1dUIppT0Z7h8+bJcq0IX3nioqqoqU/MCEyoMkrP3lvc3dGBgQNYV7rQAAGHQtAAAYdC0\nAABh0LQAAGHQtAAAYVTkjF4ZobK9kef+++83tePHj4/Ke02dOlXWly9fPirvN1ruvfdeU9u0aZNc\nO2fOnFE+m0Ls/BqU3Pnz5wv/nnM2gVQjhf72t7/Jtb/4xS9Mbd++fXKtSsh5qbfe3l5Tu/vuu+Va\nlQb2xjAV5Y2tUufb2Ngo16o03pQpU+TatWvXmtq8efMKn4O6tinpz6HGS3n1hoYGuZg7LQBAGDQt\nAEAYNC0AQBg0LQBAGLfVGKdf//rXpnb48GG5dvHixabmPRD+4x//aGq/+c1v5NrXXnvN1ObOnSvX\nnj59WtaL8vYWUmNq3nnnncLHVeGMlFJ6/vnnCx8DsakxTl64Qj1k99aqB/319fVyrQpWLViwQK6t\nq6szta6uLrn25MmTplZbWyvX9vT0mNrg4KCp9fX1ydd3dnaaWnd3t1zb0tJiaiq4klJKR44cMTVv\nPy11zWbPni3XqtCF+rwp+aELJWdPLu60AABh0LQAAGHQtAAAYdC0AABh0LQAAGHcVunBRYsWFap5\nWltbZX3dunWm9uKLL8q1Z86cMTUvPehtaFeUt5mcSg9656ASVgsXLhzReSE+b5NARaXIvE0kVapw\n5syZcu2nP/1pU5s+fXrhc/A2RVS8EUhq00qVKLx48aJ8vUoKeuOl1DXzjqsSkN5a9X45G1F6SVAl\n53v3cKcFAAiDpgUACIOmBQAIg6YFAAjjtgpilJO3p05OiCEnJJJDjZ26cOGCXPvQQw+Z2sqVK0t+\nToglZx8+tdZ7/fXr103NG0emxhp5+9hdvXrV1NRop5Ty9oxSY5TUyKecgIi3R5ban2r79u1yrfq8\nK1askGvVOCzv75f6frxQTk5oI+ffE3daAIAwaFoAgDBoWgCAMGhaAIAwaFoAgDBID45j3piaJ554\nwtS88Srf+c53TK26unpkJ4bwVOrNS4CptTmb/nljjVSizztuTipRna+30aHa3FG9Xo17SklvcOl9\nBrUp7EsvvSTXqt/zxz72Mbn27rvvNjXvfFWCsaamRq7NGc1EehAAMC7RtAAAYdC0AABh0LQAAGEQ\nxBjHtm3bJusdHR2m1tDQINfec889pTwljBNecKco7yG9CjHkBCYGBgYKv593DioI4Y0qUvtsqWuj\nRjulpPetunTpklyrfs979uyRa1etWmVqXhBDjWzyvl8VaPHk7KOW8++JOy0AQBg0LQBAGDQtAEAY\nNC0AQBg0LQBAGKQHx4mTJ0+a2saNGwu/vr29XdbVRnuASt7ljOLxqCSbStilpEcKDQ8Py7UqyaZq\nKfljoxT1fmp8mndMlUq8ePGiXLtv3z5TU9cgpZTWrFljarNmzZJr1XVQm0h67+ddc+97U3JSidxp\nAQDCoGkBAMKgaQEAwqBpAQDCIIgxTuzatcvUvIe0Tz31lKnNmzev5OeE8UuN3fFGHamxSN7YHvXw\nPmekkBcGUefQ09NT+By8z6beT9W8PbJUaOPPf/6zXHvs2DFTe+ihh+Ta1tZWU/OCJ+r61tXVybXq\nc5RiHJZXV7jTAgCEQdMCAIRB0wIAhEHTAgCEQRAjGC9c8dJLL5laVVWVXPutb33L1HIehALq4XvW\nnkjOw/scKjBRU1Mj1165cqXwOajfjTchQoUb6uvrTU1N+kgppX/84x+m9qtf/UquVb/RJ598Uq5V\n++D19fXJtdXV1abmBVpyplzkIIgBABiXaFoAgDBoWgCAMGhaAIAwaFoAgDBIDwbzox/9SNbfeOMN\nU/vc5z4n1zKyCSOlUnM5+2l5aTE1msnbi0ql/y5fvizXDg4OmpqX6FNpOo86h9raWlPzkpW/+93v\nTE39llNKadWqVab2+OOPy7Uq5ZcztsqTM2YrJxGY82+HOy0AQBg0LQBAGDQtAEAYNC0AQBgEMf6H\nHT582NSeffZZuVaNjvnmN79Z8nMCUsob2aTWens75QQx1GilnL2dvOOqcxgYGJBrGxoaTE2NWnvt\ntdfk63/5y1+a2ty5c+Xaz372s6Y2Y8YMuba3t1fWFRVS8a5NDnUdvO/H+/egcKcFAAiDpgUACIOm\nBQAIg6YFAAiDpgUACIP04P8Ab4O5devWmZpKQaWU0uc//3lTY1wT/heocT7ev+Oir09Jj2HyxgGp\nzSG9xJpKCqqEXUopDQ8Pm9qbb75palu3bpWvV5tAPvPMM3Jta2urqXnXUV2znPFUXnpQJQK9a54z\nmokxTgCAcYmmBQAIg6YFAAiDpgUACIMgRpmpkTarV6+Wa48dO2ZqixYtkmu/8Y1vjOzEgFGS85Bd\n/T6816uwgTcmSK1V45pS0sGoxsbGwsf9+c9/bmreHlnLli0ztSeeeEKubW5uNjUvxKX2vVKhkZRS\nqqurM7VSXEcV2vDCLwQxAADjEk0LABAGTQsAEAZNCwAQBk0LABAG6cEyu3Tpkqnt2bOn8Ot37Ngh\n69OmTbvVUwKyqRSYtzGkWuul0xQv9TZhgv3zpVJzHi/1psYdeSOQjh8/bmpnz541tYULF8rXb9iw\nwdTmz58v16o0nrc5pbo2apRVLvVdemO2vOur5Gw6yZ0WACAMmhYAIAyaFgAgDJoWACAMghij5PLl\ny7K+fPnywsf4yU9+YmpLly695XMCSkWFLrwRPUrOmCDvuOpBv/fwXwUTvICHCnP885//lGu3bdtm\namfOnDG1Rx99VL5e/Z7Vuaak9/TyQhC1tbWm5o1KGhoaMjXv+1Hfu7c259+DCpl4uNMCAIRB0wIA\nhEHTAgCEQdMCAIRB0wIAhEF6cJRs3bpV1k+dOlX4GI888oip5SRygNGiUmveGKeRHtej0nAqYefV\nvdFMKoF45MgRuXb37t2mptJ/CxYskK9Xa70knbq+OeOPVEowJT0KyksEqmRlzvgu77PlfO/caQEA\nwqBpAQDCoGkBAMKgaQEAwiCIUQJqT52vf/3r5T8RoExu3LhReK16qO89vFchCC98pOreflo5e2+d\nPHnS1H7/+9/LtepzLFmyxNQaGxvl63OCDSp44q1VwRNvrQpBeEEMdR2946q6F7jIGgFWeCUAAGOM\npgUACIOmBQAIg6YFAAiDpgUACIP0YAm88cYbptbb21v49YsWLZJ1b8wMMNZGa5xYzuaSKsHoJdnU\nho8XLlyQa/ft22dq7e3thY9bX19vajU1NfL1/f39pjZlyhS5ViUNvdFMOVSiz0v5qXSnlzRUacec\ncU0e7rQAAGHQtAAAYdC0AABh0LQAAGEQxCizD33oQ6bW1tYm1xLEwHiQE67IGucjAgBeKCBHQ0OD\nqX30ox+Va6uqqkzt8ccfNzVvPy3FuwYq9KHCDinpkIoawZSSDkeowIV3bt45FD0v77ge7rQAAGHQ\ntAAAYdC0AABh0LQAAGHQtAAAYVTkJD9GqGxvhNva6MwXwv/R2dlpfs85f0u8tJg6Rk7S0DsHtdZL\nsuWk9BQ1bslL7qnzyhmL5J1XzvnmrPXGZCk5iUC1tqmpSR6AOy0AQBg0LQBAGDQtAEAYNC0AQBjl\nDGIAADAi3GkBAMKgaQEAwqBpAQDCoGkBAMKgaQEAwqBpAQDCoGkBAMKgaQEAwqBpAQDCoGkBAMKg\naQEAwqBpAQDCoGkBAMKgaQEAwqBpAQDCoGkBAMKgaQEAwqBpAQDCoGkBAMKgaQEAwqBpAQDCoGkB\nAMKgaQEAwqBpAQDCoGkBAMKgaQEAwqBpAQDCoGkBAMKgaQEAwqBpAQDCmFDG93qvjO+F21fFWJ/A\n7eD8+fPm9/zee/onXlExsq/EO65y5513yvrNmzcL1bz3yzmu+rx33KHvD9R75Xze0eKdr3fNRuO4\nLS0t8h8Od1oAgDBoWgCAMGhaAIAwyvlMC8A45j27Us8xvGcjOc+/1LOfnONOmKD//Km13nMm7xlN\nUTnPr9T53rhxY0Tv751DKZ5PjvT78XCnBQAIg6YFAAiDpgUACIOmBQAIg6YFAAiD9CCAkvDSeCrh\nljMZwUuyeVMqiq71knfXr18vdF4p6c+hzlcd0zNlypTC7zU0NCTXDg4OFn6/yspKU/OujfqOc76f\nUqQSudMCAIRB0wIAhEHTAgCEQdMCAIRBEKMEfvrTn5paf3+/XHvo0CFT+8EPflD4vV544QVZf+yx\nx0zt4x//eOHjAjlyRvQU3b7DW5sTuPCoYMK5c+fk2hMnTpja6dOn5dre3l5TU6GLmpoa+fply5aZ\n2pIlS+RadQzvml+7ds3Ucq5jdXW1rOeETFSYI+d793CnBQAIg6YFAAiDpgUACIOmBQAIg6YFAAij\nImcTshEq2xuNlg0bNsj697///TKfifX+97/f1P7whz/ItVOnTh3t0xlLxefB4JadO3fO/J69MU45\n6UH190iNGfKO4aV2Ozs7Te3AgQNy7cGDB03t6NGjcq063zlz5pjaxIkT5evf9773mdoDDzwg16rf\n7eTJk+XanM0p1diouro6uVYlEL3kn0oV5pxXU1OT/EfCnRYAIAyaFgAgDJoWACAMmhYAIAyCGA4V\nuihF4GLp0qWm9uSTT8q1x48fN7Xt27cXfi9vPNSXvvSlwscIiCBGGXR2dhb+PecEMaqqqkzNC3io\nv13vvvuuXLtr1y5Ta2trk2v//ve/m9rMmTPl2gcffLDQ2kuXLsnXq2tTW1sr1zY1NZlaa2tr4bVe\nSEVdXy/goQIlXlAmZ58u9V02NzcTxAAAxEbTAgCEQdMCAIRB0wIAhEHTAgCEcdtvAvmvf/1L1n/4\nwx8WPoZKEL366qtyrdrIzRvxopI2aoO6lFLau3evqV24cEGuBUZD1kZ+zjifnI0KVSJv9+7dcu3P\nfvYzU/vLX/4i137gAx8wtdWrV8u1H/nIR0ytp6fH1NRoqJRSam9vNzVvU8U1a9aYWkNDg1yrNDY2\nyrraINOjzs1LdypeetBLkyrcaQEAwqBpAQDCoGkBAMKgaQEAwrjtgxheWEGNFVGBi5RSev31103N\nG4OSY9u2babmPdBVPvOZz4z4HAAlJ3SheA/kh4aGTM0LbXR0dJhab2+vXKvGGn3yk5+Ua1etWmVq\n3m/fC038f97n7e7uNrWBgQG59q677jI19blS8sNditpPa3h4WK5VoQ31naWk/4Z651X0OqbEnRYA\nIBCaFgAgDJoWACAMmhYAIAyaFgAgjNs+PfjAAw/IukoVesmX6urqkp7Tf6hRUl6qBygnNW4pZ4M/\nLxGoUonecdUx5s2bJ9e2tLSY2uzZs+VadQwvDXzt2rVCta6uLvn6w4cPm1pzc7Ncq66Nd1z1t0qN\nkEsppbq6OlNTm3GmpK95X1+fXKt4qVPv34NcW3glAABjjKYFAAiDpgUACIOmBQAI47YPYnimTp1a\ntvfasWOHrL/11luFj7Fy5UpTu++++275nID/ZqR7Z6lwhlf39tiaPn26qX3wgx+Ua/v7+03N2wdK\n7VHlBTGuXLliahcvXjQ1L0A1c+ZMU3v44Yfl2vnz55uaugYp6bFI6hqkpK9vfX29XKsCGqUIV6jw\nioc7LQBAGDQtAEAYNC0AQBg0LQBAGDQtAEAYpAfL7M033zS1L3/5y3Kt2lxtxowZcu2WLVtMrbKy\nMvPsgFvnpfxU3UvTqRRZRUWFXKsSfd6oNfV+XkJOjWXzPtuJEydM7ciRI6bmbTarEoFLly6Va9UY\npqtXr8q1Kv3spQfVmCwvEaiugzfGTl3znJFcHu60AABh0LQAAGHQtAAAYdC0AABhEMQos/b2dlNT\ngQvP+vXrZX3BggW3fE5AKXijmdTD95wH797DezWqyAtMNDY2mpp3vurc3n77bbl2+/btpnby5ElT\n84IYa9asMbVFixbJtSpc4X3egYGBwmvVdeju7pZra2trTc0LyuR8P953rHCnBQAIg6YFAAiDpgUA\nCIOmBQAIg6YFAAiD9OAoefrpp2V9586dhY/xla98xdS+9rWv3fI5AaXiJe+K8tKDkyZNMjUvWdbb\n22tqTU1Ncq0aNeSNklLjjg4cOCDXdnR0mJpKyD322GPy9Y8++qipeRs7qs/gjXFS46xU8i8lvbFj\nT0+PXKvGO3njsIq+PiV/Q06FOy0AQBg0LQBAGDQtAEAYNC0AQBgEMUrgypUrpvbKK6/ItYODg6bW\n3Nws127evNnUvP2CgHLyRvco6uG793oV8FB7bKWkH957Y4IUb7+5U6dOmdrRo0fl2ilTppiaGsP0\niU98Qr5+8eLFpuaFFdRoJu8zqECLGgOVkr7m6vUp6fCLCnJ4x/V437HCnRYAIAyaFgAgDJoWACAM\nmhYAIAyaFgAgDNKDJfDUU0+ZWmdnZ+HXP/fcc7I+bdq0Wz4noNxyNlX0qA1RvUSgl5wruravr0+u\n3b17t6nt27dPrlXHmDx5sqmpEUwp6UTfxYsX5Vp1Hb3RTOo6et+PSiSrVGRKOsGoEtHe+3njmnL+\njXCnBQAIg6YFAAiDpgUACIOmBQAIgyBGhkOHDsn6nj17Ch9j7dq1prZx48ZbPSVgTKiH7F5gQo0l\n8vbIUrwQgxoF5YUC1Ki1/fv3y7WHDx82NS9AcN9995na7NmzTW3GjBny9eraXLp0Sa5VdTUGKiUd\npPD2D1PX1/su1fXNGcnlHTdnLBh3WgCAMGhaAIAwaFoAgDBoWgCAMAhiOK5evWpqmzZtkmu9B5zK\nsmXLTI09shCNenDu7QOVMxlBBTS8h/cqXOFNojl+/LiptbW1ybUqoKECFymltHz5clP71Kc+ZWpN\nTU3y9d3d3abW09Mj16pJGV1dXXKtCoOoaRYp6e/CC7So78L7LtW/B28qB0EMAMC4RNMCAIRB0wIA\nhEHTAgCEQdMCAIRBetDxve99z9R++9vfFn79008/LeuMbMJ4oFJgOXsieYlAlaT1Uom9vb2mdurU\nKbl27969ptbe3i7XXrt2zdTuueceuXb69OmFXn/69Gn5epUU9NKDs2bNMrVJkybJteo6eilnlSr0\nkoZq5FPWXljOWi9VKI9ReCUAAGOMpgUACIOmBQAIg6YFAAiDIIZj8+bNI3r9t7/9bVlnZBPGKy8w\noXhBDCXnIb23F9Wf/vQnUzt79qxc+/DDD5vaI488ItequgqIeO91/fp1U6urq5Nr1WimxsZGuVZd\nXy/gce7cOVPr7++Xa1XwwxvBNG3aNFOrr6+XaxnjBAAYl2haAIAwaFoAgDBoWgCAMGhaAIAwSA+O\nErVBXUp5I09yVFVVmZqX0FIb7Q0NDRV+L7VBZkopbdmypfAxFO98VZKzsrJyRO+FkVH/jr0EWM6/\nN3UM79+FSqJ5x/3rX/9qal7acf78+aa2ZMkSubavr8/UDh48aGpqA8eUUpozZ46peQlIlR70EoFq\nE8dXXnlFrj127JipeSOf1OdVKcGUUlq5cqWptba2yrXq75eHOy0AQBg0LQBAGDQtAEAYNC0AQBgE\nMUaJ2vtmNK1fv97UZs6cKdd2dHSY2ne/+92Sn1OpqGv5zDPPjMGZ4D/UaCUv2KDq3loVxFChAo8X\nClCjkc6fPy/Xqt9HW1ubXHvmzBlTU/tpTZig/9S+8847puaFSVQIQo2MSkl/tpdfflmuzRklpY7r\njbhSvCBazggw7rQAAGHQtAAAYdC0AABh0LQAAGHQtAAAYVTkbLA2QmV7o1JQ6bStW7eOwZmMLZV6\nytnA74tf/KKsq432PB/+8IdNbd68ed7y4rvJ4ZZ1dnaa33NOIjDn787ly5dlXaXsvPFDO3fuNDW1\nMWRKOuF21113ybUqaajSg6qWkt5ssba2Vq5taGgwNW9cXM71bW5uLnwOijfiasWKFabmJZrV35Sm\npib5W+ZOCwAQBk0LABAGTQsAEAZNCwAQBkGMDD/+8Y9l3Xv4W9Rbb70l6yMdrfTVr35V1tV+QZ41\na9aYWlNT0y2fUxkQxCiDjo4O83v2RvSoIIYX2pg4caKpeWOcVEDD+3umxh2pEEVKKZ07d67Qe6WU\n0rvvvmtqKlwxefJk+XoVePBGKOUEoFS4whstp74fb+yU+lvX2Ngo16rwSs6+WS0tLQQxAACx0bQA\nAGHQtAAAYdC0AABh0LQAAGGQHsR4Q3qwDLq6ukY0xknVUtLpQc/Vq1cLr1XJO29U0Y0bN0zNS0aq\nutpUUW3g6L3eey81ssm7jqo+adIkuVbxvkuV5KysrJRr1TX3PptCehAAEB5NCwAQBk0LABAGTQsA\nEIae1QEA/4UKG+Q8ZPeofae846oQmRcgUGOJVOAiJR3wUHt3paTDHDlhEvV677zUWCTvvNTYKo8X\npFCqq6tNzQuDqO/HC/7l/NvhTgsAEAZNCwAQBk0LABAGTQsAEAZNCwAQBulBANlyRjOpRJ+3VlFJ\nxZT8lF3RY6ikokel5lLSqbfu7m5T81KNPT09hd9LjWHyNoZUde8cFC/N530XivqOvfPNGSfInRYA\nIAyaFgAgDJoWACAMmhYAIAyCGABKImc/rVLs45cTNsgZO6UCD144Ynh42NTU580Z7ZQTgvDWeoGH\nou/nhVTUWu+7VNfBC8/kBHO40wIAhEHTAgCEQdMCAIRB0wIAhEHTAgCEQXoQQLbRSgQWfS+Pt6Fh\nzoaRKhHojS9Sxx1pwi7nvTw5Y7ZUoi9nrZdgVHUvPcgYJwDAuETTAgCEQdMCAIRB0wIAhFExWg9P\nAQAoNe60AABh0LQAAGHQtAAAYdC0AABh0LQAAGHQtAAAYdC0AABh0LQAAGHQtAAAYdC0AABh0LQA\nAGHQtAAAYdC0AABh0LQAAGHQtAAAYdC0AABh0LQAAGHQtAAAYdC0AABh0LQAAGHQtAAAYdC0AABh\n0LQAAGH8Gwk+Z4j/7+iaAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f883c559550>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs, \"./my_model_sparse.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Variational Autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "n_inputs = 28*28\n",
    "n_hidden1 = 500\n",
    "n_hidden2 = 500\n",
    "n_hidden3 = 20  # codings\n",
    "n_hidden4 = n_hidden2\n",
    "n_hidden5 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.001\n",
    "\n",
    "activation = tf.nn.elu\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer(mode=\"FAN_AVG\",\n",
    "                                                             uniform=True)\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, n_inputs])\n",
    "\n",
    "weights1 = tf.Variable(initializer([n_inputs, n_hidden1]))\n",
    "weights2 = tf.Variable(initializer([n_hidden1, n_hidden2]))\n",
    "weights3_mean = tf.Variable(initializer([n_hidden2, n_hidden3]))\n",
    "weights3_log_sigma = tf.Variable(initializer([n_hidden2, n_hidden3]))\n",
    "weights4 = tf.Variable(initializer([n_hidden3, n_hidden4]))\n",
    "weights5 = tf.Variable(initializer([n_hidden4, n_hidden5]))\n",
    "weights6 = tf.Variable(initializer([n_hidden5, n_inputs]))\n",
    "\n",
    "biases1 = tf.Variable(tf.zeros([n_hidden1], dtype=tf.float32))\n",
    "biases2 = tf.Variable(tf.zeros([n_hidden2], dtype=tf.float32))\n",
    "biases3_mean = tf.Variable(tf.zeros([n_hidden3], dtype=tf.float32))\n",
    "biases3_log_sigma = tf.Variable(tf.zeros([n_hidden3], dtype=tf.float32))\n",
    "biases4 = tf.Variable(tf.zeros([n_hidden4], dtype=tf.float32))\n",
    "biases5 = tf.Variable(tf.zeros([n_hidden5], dtype=tf.float32))\n",
    "biases6 = tf.Variable(tf.zeros([n_inputs], dtype=tf.float32))\n",
    "\n",
    "hidden1 = activation(tf.matmul(X, weights1) + biases1)\n",
    "hidden2 = activation(tf.matmul(hidden1, weights2) + biases2)\n",
    "\n",
    "hidden3_mean = tf.matmul(hidden2, weights3_mean) + biases3_mean\n",
    "hidden3_log_sigma = tf.matmul(hidden2, weights3_log_sigma) + biases3_log_sigma\n",
    "noise = tf.random_normal(tf.shape(hidden3_log_sigma), dtype=tf.float32)\n",
    "hidden3 = hidden3_mean + tf.sqrt(tf.exp(hidden3_log_sigma)) * noise\n",
    "\n",
    "hidden4 = activation(tf.matmul(hidden3, weights4) + biases4)\n",
    "hidden5 = activation(tf.matmul(hidden4, weights5) + biases5)\n",
    "logits = tf.matmul(hidden5, weights6) + biases6\n",
    "outputs = tf.sigmoid(logits)\n",
    "\n",
    "reconstruction_loss = tf.reduce_sum(tf.nn.sigmoid_cross_entropy_with_logits(labels=X, logits=logits))\n",
    "latent_loss = 0.5 * tf.reduce_sum(tf.exp(hidden3_log_sigma) + tf.square(hidden3_mean) - 1 - hidden3_log_sigma)\n",
    "cost = reconstruction_loss + latent_loss\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "training_op = optimizer.minimize(cost)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "from functools import partial\n",
    "\n",
    "n_inputs = 28*28\n",
    "n_hidden1 = 500\n",
    "n_hidden2 = 500\n",
    "n_hidden3 = 20  # codings\n",
    "n_hidden4 = n_hidden2\n",
    "n_hidden5 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.001\n",
    "\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "my_dense_layer = partial(\n",
    "    tf.layers.dense,\n",
    "    activation=tf.nn.elu,\n",
    "    kernel_initializer=initializer)\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, n_inputs])\n",
    "hidden1 = my_dense_layer(X, n_hidden1)\n",
    "hidden2 = my_dense_layer(hidden1, n_hidden2)\n",
    "hidden3_mean = my_dense_layer(hidden2, n_hidden3, activation=None)\n",
    "hidden3_gamma = my_dense_layer(hidden2, n_hidden3, activation=None)\n",
    "noise = tf.random_normal(tf.shape(hidden3_gamma), dtype=tf.float32)\n",
    "hidden3 = hidden3_mean + tf.exp(0.5 * hidden3_gamma) * noise\n",
    "hidden4 = my_dense_layer(hidden3, n_hidden4)\n",
    "hidden5 = my_dense_layer(hidden4, n_hidden5)\n",
    "logits = my_dense_layer(hidden5, n_outputs, activation=None)\n",
    "outputs = tf.sigmoid(logits)\n",
    "\n",
    "reconstruction_loss = tf.reduce_sum(tf.nn.sigmoid_cross_entropy_with_logits(labels=X, logits=logits))\n",
    "latent_loss = 0.5 * tf.reduce_sum(tf.exp(hidden3_gamma) + tf.square(hidden3_mean) - 1 - hidden3_gamma)\n",
    "cost = reconstruction_loss + latent_loss\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "training_op = optimizer.minimize(cost)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train cost: 18604.3 \tReconstruction loss: 14818.0 \tLatent loss: 3786.32\n",
      "1 Train cost: 16815.6 \tReconstruction loss: 13203.3 \tLatent loss: 3612.25\n",
      "2 Train cost: 16529.8 \tReconstruction loss: 12717.1 \tLatent loss: 3812.65\n",
      "3 Train cost: 16443.3 \tReconstruction loss: 12582.1 \tLatent loss: 3861.12\n",
      "4 Train cost: 16256.6 \tReconstruction loss: 12489.2 \tLatent loss: 3767.41\n",
      "5 Train cost: 15971.0 \tReconstruction loss: 12240.5 \tLatent loss: 3730.49\n",
      "6 Train cost: 15806.8 \tReconstruction loss: 11980.4 \tLatent loss: 3826.42\n",
      "7 Train cost: 16334.4 \tReconstruction loss: 12480.7 \tLatent loss: 3853.68\n",
      "8 Train cost: 15835.0 \tReconstruction loss: 12082.8 \tLatent loss: 3752.25\n",
      "9 Train cost: 15549.4 \tReconstruction loss: 11733.3 \tLatent loss: 3816.08\n",
      "10 Train cost: 15970.6 \tReconstruction loss: 12221.3 \tLatent loss: 3749.32\n",
      "11 Train cost: 15287.9 \tReconstruction loss: 11545.8 \tLatent loss: 3742.02\n",
      "12 Train cost: 15450.1 \tReconstruction loss: 11634.0 \tLatent loss: 3816.12\n",
      "13 Train cost: 15865.1 \tReconstruction loss: 11956.3 \tLatent loss: 3908.8\n",
      "14 Train cost: 15065.9 \tReconstruction loss: 11274.5 \tLatent loss: 3791.37\n",
      "15 Train cost: 15817.3 \tReconstruction loss: 11941.5 \tLatent loss: 3875.85\n",
      "16 Train cost: 15068.3 \tReconstruction loss: 11291.6 \tLatent loss: 3776.76\n",
      "17 Train cost: 15378.0 \tReconstruction loss: 11527.9 \tLatent loss: 3850.08\n",
      "18 Train cost: 15467.5 \tReconstruction loss: 11681.3 \tLatent loss: 3786.2\n",
      "19 Train cost: 15028.8 \tReconstruction loss: 11369.0 \tLatent loss: 3659.77\n",
      "20 Train cost: 14593.2 \tReconstruction loss: 10931.3 \tLatent loss: 3661.84\n",
      "21 Train cost: 15184.6 \tReconstruction loss: 11463.8 \tLatent loss: 3720.81\n",
      "22 Train cost: 15549.5 \tReconstruction loss: 11719.6 \tLatent loss: 3829.93\n",
      "23 Train cost: 15120.7 \tReconstruction loss: 11356.6 \tLatent loss: 3764.11\n",
      "24 Train cost: 15043.3 \tReconstruction loss: 11271.0 \tLatent loss: 3772.25\n",
      "25 Train cost: 14868.0 \tReconstruction loss: 11159.2 \tLatent loss: 3708.84\n",
      "26 Train cost: 15204.3 \tReconstruction loss: 11325.8 \tLatent loss: 3878.51\n",
      "27 Train cost: 14795.7 \tReconstruction loss: 11049.2 \tLatent loss: 3746.59\n",
      "28 Train cost: 14319.1 \tReconstruction loss: 10681.5 \tLatent loss: 3637.58\n",
      "29 Train cost: 15113.9 \tReconstruction loss: 11406.9 \tLatent loss: 3707.06\n",
      "30 Train cost: 14934.5 \tReconstruction loss: 11208.4 \tLatent loss: 3726.05\n",
      "31 Train cost: 15178.9 \tReconstruction loss: 11462.5 \tLatent loss: 3716.41\n",
      "32 Train cost: 14835.1 \tReconstruction loss: 11094.8 \tLatent loss: 3740.24\n",
      "33 Train cost: 14892.1 \tReconstruction loss: 11055.8 \tLatent loss: 3836.29\n",
      "34 Train cost: 14663.1 \tReconstruction loss: 10960.1 \tLatent loss: 3703.0\n",
      "35 Train cost: 14606.7 \tReconstruction loss: 10958.7 \tLatent loss: 3648.02\n",
      "36 Train cost: 14877.4 \tReconstruction loss: 11219.5 \tLatent loss: 3657.99\n",
      "37 Train cost: 14887.5 \tReconstruction loss: 11089.1 \tLatent loss: 3798.37\n",
      "38 Train cost: 14376.7 \tReconstruction loss: 10733.5 \tLatent loss: 3643.15\n",
      "39 Train cost: 14774.7 \tReconstruction loss: 11164.5 \tLatent loss: 3610.25\n",
      "40 Train cost: 14465.2 \tReconstruction loss: 10808.1 \tLatent loss: 3657.06\n",
      "41 Train cost: 14554.3 \tReconstruction loss: 10781.6 \tLatent loss: 3772.74\n",
      "42 Train cost: 14828.5 \tReconstruction loss: 11083.7 \tLatent loss: 3744.73\n",
      "43 Train cost: 14663.9 \tReconstruction loss: 10968.0 \tLatent loss: 3695.93\n",
      "44 Train cost: 14264.3 \tReconstruction loss: 10717.4 \tLatent loss: 3546.83\n",
      "45 Train cost: 14479.3 \tReconstruction loss: 10833.3 \tLatent loss: 3645.97\n",
      "46 Train cost: 14839.5 \tReconstruction loss: 11099.8 \tLatent loss: 3739.75\n",
      "47 Train cost: 14417.5 \tReconstruction loss: 10806.8 \tLatent loss: 3610.71\n",
      "48 Train cost: 14933.3 \tReconstruction loss: 11270.8 \tLatent loss: 3662.51\n",
      "49 Train cost: 15005.8 \tReconstruction loss: 11272.2 \tLatent loss: 3733.64\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 50\n",
    "batch_size = 150\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = mnist.train.num_examples // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch})\n",
    "        cost_val, reconstruction_loss_val, latent_loss_val = sess.run([cost, reconstruction_loss, latent_loss], feed_dict={X: X_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"Train cost:\", cost_val, \"\\tReconstruction loss:\", reconstruction_loss_val, \"\\tLatent loss:\", latent_loss_val)\n",
    "        saver.save(sess, \"./my_model_variational.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Encode:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "n_digits = 3\n",
    "X_test, y_test = mnist.test.next_batch(batch_size)\n",
    "codings = hidden3\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_variational.ckpt\")\n",
    "    codings_val = codings.eval(feed_dict={X: X_test})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Decode:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_variational.ckpt\")\n",
    "    outputs_val = outputs.eval(feed_dict={codings: codings_val})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Let's plot the reconstructions:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAHGCAYAAABaaN0mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGHBJREFUeJzt3WuMXVX5P/A9vVJL6QUcyq0URFqoii3BgoAY5JoYUQkh\nCIoBUwgRDEQkYFCaaHjhC8UXRgkIogQREKsmQJRw03ApNw1QZLi1pWnBVnulLdNpf2/+/19+a63d\nnpmn55yZOfP5vHs2a89ebdjn232eWWt37dixowKAgRo12BMAYHgSIACECBAAQgQIACECBIAQAQJA\niAABIESAABAiQAAIESAAhIxp47XsmTKydA32BGgp9/PIUns/ewIBIESAABAiQAAIESAAhAgQAEIE\nCAAhAgSAEAECQIgAASBEgAAQIkAACBEgAIQIEABCBAgAIQIEgBABAkCIAAEgRIAAECJAAAgRIACE\nCBAAQgQIACECBIAQAQJAiAABIESAABAiQAAIESAAhAgQAELGDPYEWuGpp55K6ptuuimpDzjggOKc\nCRMmJPWFF15YjJk2bdoua2Bg+vr6knrNmjXFmPx+fvnll5N6n332Kc5Zv359Un/uc58rxhxyyCFJ\nnX8GVFVVjRmTfkR2dXUVYyK2b9/ecEx+rUb1YPAEAkCIAAEgRIAAENK1Y8eOdl2rbReaNWtWUvf0\n9DTl506ePDmpjz322Kb83GaZOXNmUl977bVJPWPGjDbOphr8L2hppQHfz3WfNXkv4J133inGXH31\n1Un94IMPJvWWLVsaXquuvzFqVPrv57FjxxZj8vPy3kqdPffcM6n32muvYsz48eOTuq6fesIJJyT1\nt7/97aSeNGlSw7k0Ue397AkEgBABAkCIAAEgRIAAENKRTfQlS5Yk9YsvvpjUc+bMKc7JFyc9/fTT\nxZhFixYl9dKlS4sx+eKkt956a9eTrZEvXqqqqtpvv/2Sevny5Q1/zo033pjU11xzzYDnshs00Ttb\nU+7nvIlet8AuX0j4gx/8IKnffPPN4pyNGzfusq6qqtq6dWtS5031qqqq3t7eXc6v7vMzX+B34IEH\nFmNy//73v4tjeQP/d7/7XVLXLY5s4eJCTXQAmkeAABAiQAAI6cgeSKvkC5befvvtYkzeA6n7fraR\ncePGFcfyHkh+naoqv0e9//77k/qss84a8Fx2gx5IZ2vKQsL+fGefb7jYqK6qsr+xdu3aYkzeY6hb\nkPjuu+8m9ebNm5O6rsd56KGHJnV3d3cx5pZbbknqn/zkJ8WY3J133pnU5557bsNzmkgPBIDmESAA\nhAgQAEIECAAhHflGwlbZY489knr27NkNzzniiCOacu18YePq1auLMfPnz0/q0047rSnXhsE0evTo\nXdZ18gb5lClTQtc++OCDd/nfTzrppIY/o25xZN3Ov7l8V9/jjjuu4Tnt5gkEgBABAkCIAAEgRA9k\nCNq0aVNx7Etf+lJS132vmi9GqnsLGwyWFm70N6TV3av5WxXrxuT9lQMOOKC5E2sCTyAAhAgQAEIE\nCAAhAgSAEE30Iej2228vjq1atSqp995772JMo0VPQPu99957xbGenp6knjhxYjFm4cKFSd2fBZTt\n5gkEgBABAkCIAAEgRA9kCHjjjTeS+qqrrmp4zpNPPlkcmz59etPmBMTkiwIXLFhQjMnfznjqqacW\nYz7+8Y83d2It4AkEgBABAkCIAAEgRA9kCPjTn/6U1L29vcWYc845J6kPPfTQls4JiFm8eHFSP/74\n48WYqVOnJvWtt95ajBmK6z5ynkAACBEgAIQIEABCBAgAIZrobVbXIL///vuTevz48cWYG2+8MamH\nQ4MNOt3WrVuLY2eddVZSb9mypRjzve99L6mnTJnS3Im1iScQAEIECAAhAgSAED2QNqtbMPTEE08k\n9Ve+8pVijIWDMPjyjRLPPPPMYsy7776b1Pvtt18x5pvf/GZSjxo1PP8tPzxnDcCgEyAAhAgQAEIE\nCAAhXfmbsVqobRcaSl588cWkPuaYY4oxkyZNSupnn322GDMMm+hdgz0BWmpE3s/5L7ycfPLJxZiu\nrvR//brdeI899tjmTqz1au9nTyAAhAgQAEIECAAhFhI22ebNm5P6vPPOS+q+vr7inPPPPz+ph2G/\nAzpSvlnixRdf3PCcyy67LKk/9alPNXVOQ4knEABCBAgAIQIEgBABAkCIhYS7Id+Zs6qq6pRTTknq\nRx99NKmPOOKI4px8cdK0adN2f3KDz0LCztZx93PdZ+Hll1+e1DfffHNSH3XUUcU5+cLBCRMmNGF2\ng85CQgCaR4AAECJAAAjRA9kNq1evLo51d3fv8py6jRLnzZvXtDkNIXogna3j7ucVK1YUx/Ke5bZt\n25L6pZdeKs7p0IXAeiAANI8AASBEgAAQYjPFAVi3bl1S9+elML/5zW+Seu7cuU2dExCT9zO++MUv\nFmPef//9pD7jjDOS+uCDD27+xIYRTyAAhAgQAEIECAAhAgSAEE30AbjtttuS+s0332x4zgknnJDU\nXV3W18FQ0NPTk9RLliwpxuT36/XXX5/Uo0ePbv7EhhFPIACECBAAQgQIACF6IDuRfz9aVVV1ww03\ntH8iwG7r7e0tjuX9jC1bthRj8oWCc+bMae7EhjlPIACECBAAQgQIACECBIAQTfSdeOKJJ4pj69ev\nb3he/gazCRMmNG1OQP/kb1p9++23izEvvPBCUk+aNKkY861vfSupJ06cuPuT6yCeQAAIESAAhAgQ\nAEL0QHbDpz/96eLYX/7yl6TWA4HBt2nTpuJY3vOYOnVqMWbs2LEtm1Mn8AQCQIgAASBEgAAQIkAA\nCOnKF9y0UNsuxJDg1YudbUjfz/nnWl9fXzFm2bJlSb127dpizCc/+cmkHjVqxP6bu/Z+HrF/GwDs\nHgECQIgAASCknT0QADqIJxAAQgQIACECBIAQAQJAiAABIESAABAiQAAIESAAhAgQAEIECAAhAgSA\nEAECQIgAASBEgAAQIkAACBEgAIQIEABCBAgAIQIEgBABAkCIAAEgRIAAECJAAAgRIACEjGnjtXa0\n8VoMvq7BngAt5X4eWWrvZ08gAIQIEABCBAgAIQIEgBABAkCIAAEgRIAAECJAAAgRIACECBAAQgQI\nACHt3AtrWLnzzjuLY5s2bUrq5557rhhz88037/LnXn/99cWxk08+Oak/+9nP9mOGQFVV1Y4d5bZc\n27ZtS+oVK1YUY2688cakfuaZZ4oxGzZsSOpJkyYl9QknnFCcM2vWrKResGBBMWbcuHHFseHIEwgA\nIQIEgBABAkCIAAEgpKuuAdUiQ/oFNJdddllS/+IXv2jbtY888sik/tvf/laMmTx5crum0yxeKNXZ\nBu1+3r59e1Jv3LixGPPXv/41qX/0ox8VY15++eWk3rx5czGmr68vqUePHp3UY8eOLc758Ic/nNSP\nP/54MWbGjBlJ3dU15G8XL5QCoHkECAAhAgSAkBG5kDDvd1RVrOcxd+7c4tjZZ5+d1D09PUn9q1/9\nqjjnlVdeSep77723GHPxxRcPeH7QCfI+7cqVK5P6iiuuKM556KGHkrquvzFx4sSk3n///Ysx06ZN\nS+p169Yl9apVq4pz8sWHjzzySDHma1/7WlIPgx5ILU8gAIQIEABCBAgAIQIEgJAR0URftmxZUt9y\nyy0NzznmmGOKYw8++GBSf+hDHyrG5Lts5guRXn/99eKcv//970m9evXqhvODkSK/h+65556kfvXV\nV4tz8sb78ccfX4zJPwfyxX1VVVWjRqX/xs7vzWuuuaY457HHHkvqvOnfSTyBABAiQAAIESAAhIyI\nHkj+vWXdBpJ5zyPfjK2qqmrPPfcc8LVvv/32pF68eHHDc84666wBXwc6Vd4DOfPMM5P6kEMOKc6Z\nPn16Un/sYx8rxkyYMCGp835HVZWfFVOnTk3qun7lf//736TOFx92Ek8gAIQIEABCBAgAISOiBzJv\n3rykrvveMl+/kX8/GpX/rvkHH3zQlJ8LI0X+0qaPfOQjSV3XA8k3Jxwzpvyoi2xgmK9BqeuV5i+8\n2nvvvZty7aHIEwgAIQIEgBABAkCIAAEgpKtuUV2LtO1Cg+nXv/51Ul9yySVJvXXr1uKc0047Lan/\n+Mc/FmPyRuIw0BldQnamfR8cTfiMijat169fn9SzZ89O6rqNEru7u5N6yZIlxZj8TYfDQO1foCcQ\nAEIECAAhAgSAkBGxkLBVXnjhheJYo57HfvvtV5xz0003JfUw7HdAy7Rq0V3eW+nt7S3GLFy4MKnX\nrFmT1HUvlbv55puTOt+AsZN4AgEgRIAAECJAAAgRIACEaKLvhieffLI4VrdQ8P+69NJLi2OHH354\n0+YE9G/x4bZt25L6t7/9bTHml7/8ZVLnu/p+/vOfL845/fTTk7pTdt6t4wkEgBABAkCIAAEgRA9k\nAC666KKkvvvuuxuec+WVVyb1d77znabOCWisbpHgww8/nNTXX399MWbjxo1JfdhhhyX1DTfcUJwz\nkhYCewIBIESAABAiQAAIESAAhGii70TePKuqqnrggQeSesuWLcWYfffdN6mvu+66pB43blwTZgf8\nf3WLBvNFgu+8804xJm+Ar1ixohiTLwL88pe/nNQzZsxoeE4n8wQCQIgAASBEgAAQogeyE+ecc05x\n7L333mt43hVXXJHU06ZNa9qcgFJdD2TTpk1JfeuttxZjenp6knrUqPLf03PmzEnqfGHw+PHji3Py\nHkjd/DqlT+IJBIAQAQJAiAABIEQP5P957rnnkvrRRx9teE7+O+FVVVVXXXVVs6YEBL322mtJ/dBD\nDxVj8rUi+UaJVVVV9913X1JPmTIlqUePHl2c0yn9jf7wBAJAiAABIESAABAiQAAIGZFN9M2bNxfH\nrr322qT+4IMPGv6co48+ujhms0RorXxh3oYNG4oxP/3pT5P61VdfLcZMnjw5qeveSLj//vsndd40\nb2fDPP9z92eBYqvn5wkEgBABAkCIAAEgZET2QH7+858Xxx5++OGG51100UVJbdEgtF/+3f9TTz1V\njFm0aFFS17387cQTT0zq008/vRiT9zzq+g65dvVF8oWQVdW/+ebzq9tEsr9/Bk8gAIQIEABCBAgA\nIQIEgJCu/jSFmqRtF2pkwoQJxbH+LBxct25dUu+5555Nm1MHGjlbko5Mg3Y/v//++0n99a9/vRjz\n+9//PqnHjh1bjPnxj3+c1BdeeGExplEzue6/jxmT/m5SXZO6P/LP5vwXAZYuXVqcs3LlyqRev359\nMeb4449P6rq3ptbsMlz7F+EJBIAQAQJAiAABIGRELiSM2rhxY1JHv9vMjR8/Pqnr3nLW19eX1Fu3\nbm34c/NNI2+66abA7Mr5XHfddcWYuu+YoRnyXsDzzz+f1IsXLy7Oyf9/POigg4ox8+bNS+q6hXlr\n1qxJ6rVr1yZ1fl9WVdkbresx5POr+yz55z//mdS33XZbUr/++uvFORMnTkzq6dOnF2M++tGPJnX+\n+VNVVTVp0qSk3lkvyBMIACECBIAQAQJAiAABIEQTfQAOOOCAlvzcSy+9NKnzt6BVVVWtWrUqqX/2\ns5+1ZC79Uff38I1vfGMQZsJI1N3dndR1v3SSN4brmsB33XVXUtf9f71kyZKkXr58eVLXvYE0b7zX\nLebbvn17UufN76oqG/b5z61r4M+aNSupZ86cWYzJfymhroluN14AWkqAABAiQAAIGZGbKdZ9X58v\n0hmO8k3c6r4bzuUb0R133HENz8k3Y6uqqjr00EPzQzZT7Gzt++DIPqM2bdqU1HfccUdxTn4s7yHW\n/ZzJkycXY/IFfnk/o7e3tzgn703kC3qrquyB1F07X2yYXytfEFhVVXXSSScl9dFHH12M+cIXvpDU\ndT2QGjZTBKB5BAgAIQIEgJAR2QOpk39n2p8XTNX5xz/+kdSR9RpXX311ceywww5reF7+3Wb++/Jt\npgfS2Qbtfs77B3U9htWrVyf1n//852LMU089ldR1fYi8n5G/VC7fdLCqys+OvNdSVVW11157JXXd\n2q+8fzFnzpyk3mOPPYpz8n5G3VqRffbZJ6nz3ulO6IEA0DwCBIAQAQJAiAABIEQTnVbRRO9sQ/p+\nzj/Xop9z+aaCkZ/b340Jc81642mTaKID0DwCBIAQAQJAiB4IraIH0tnczyOLHggAzSNAAAgRIACE\nCBAAQgQIACECBIAQAQJAiAABIESAABAiQAAIESAAhAgQAEIECAAhAgSAEAECQIgAASBEgAAQMqaN\n1/KGOugc7mc8gQAQI0AACBEgAIQIEABCBAgAIQIEgBABAkCIAAEgRIAAECJAAAgRIACECBAAQgQI\nACECBIAQAQJAiAABIESAABAiQAAIESAAhAgQAEIECAAhAgSAEAECQIgAASBkTBuvtaON12LwdQ32\nBGgp9/PIUns/ewIBIESAABAiQAAIESAAhAgQAEIECAAhAgSAEAECQIgAASBEgAAQIkAACBEgAIQI\nEABCBAgAIQIEgBABAkBIO18oxU6sXr06qbu7u4sx99xzT1KfffbZLZ0T0Fhvb29x7K677krqf/3r\nX8WY888/P6lnz55djBk1auj/+37ozxCAIUmAABAiQAAI0QMZAvLvSOu++zzwwAPbNR2gn/r6+opj\nixYtSup33323GHPuuecmdVdXV3Mn1iaeQAAIESAAhAgQAEIECAAhmuhDwNNPP53UkyZNKsbMnz+/\nXdMB+mn9+vXFsbfeeiupx40bV4yZPn16UmuiAzCiCBAAQgQIACF6IG22cuXK4tj3v//9pL7yyivb\nNR1gALZv357Uy5cvL8aMHj06qT/zmc8UY6ZOndrciQ0STyAAhAgQAEIECAAheiBttnTp0uLYpk2b\nkvqCCy5o13SAAdi2bVtS5y+PqqryJVOXXXZZMWbMmM746PUEAkCIAAEgRIAAECJAAAjpjE7OMPLd\n7363OHbYYYcl9cyZM9s0G2BXduzYkdTr1q1L6meeeaY4J79/u7u7izHDdfPEnCcQAEIECAAhAgSA\nED2QFlu7dm1SP/LII8WYT3ziE0ld9wIaYPDlL4vq6ekpxixYsCCpO2XRYB1PIACECBAAQgQIACEC\nBICQzu3uDBHPP/98wzEHHXRQG2YCDFT+BsJFixYldb6TdlVV1dFHH53U+RsKO4knEABCBAgAIQIE\ngBA9kBZbvHhxwzELFy5sw0yAgcrfLvjAAw8k9ahR5b/B9UAAoAEBAkCIAAEgRIAAENKVv3Grhdp2\nocH05ptvJvX8+fOT+sgjjyzOefjhh5O6Q3bv7IxXrrEzI+J+XrVqVVLni35nz55dnJP/4swee+zR\n/Im1X+397AkEgBABAkCIAAEgpCO+bB9K8n7G6tWrk/qoo44qzumQngd0nLfffjupt23bltT5osGq\nGllvFPUEAkCIAAEgRIAAEOLL9yZ79tlnk7qrK/316QsuuKCd0wH6qa+vrzh2xx13JHV+P19++eXF\nOXUbLHaqkfMnBaCpBAgAIQIEgBABAkCIzRR3w8aNG4tjs2bNSuru7u6kfuGFF1o6pyHEZoqdrePu\n5w0bNhTHZsyYkdT5ot98oWFVVdXEiRObOq8hwmaKADSPAAEgRIAAEGIh4W649957i2MrV65M6vPO\nO69d0wEGIO//vvrqq8WY9evXJ/U555yT1BMmTGj+xIYRTyAAhAgQAEIECAAhAgSAEE303fDGG280\nHLP33nu3YSbAQOW77/7hD38oxuQLB0888cSkHkk779YZ2X96AMIECAAhAgSAEJsp7oaZM2cWx5Yt\nW5bUr7zySlLPnj27lVMaSmym2NmG/f28devWpD711FOLMS+99FJSv/baa0m9zz77NH9iQ5PNFAFo\nHgECQIgAASBEgAAQYiHhAPT09CT1ihUrBmkmwO7K7+e8QV5VVbXvvvsm9ZQpU1o6p+HGEwgAIQIE\ngBABAkCIHsgA3HfffUmdb8ZWVeVma4cffnhL5wQ0Vnev/vCHP0zq//znP8WYU045JalHjx7d3IkN\nc55AAAgRIACECBAAQvRAdqK3t7c4dvfddzc878ILL0zqkf7CGRgK6u7nxx57LKm7usr9Ai+55JKW\nzakT+HQDIESAABAiQAAIESAAhGii70Rd83v69OlJPXfu3GLMV7/61ZbNCYipWwB4xhlnJPWzzz5b\njOnu7m7ZnDqBJxAAQgQIACECBICQrh07drTrWm27EENCuSqLTjLs7+f8s6/us9BC4P9Vez/72wEg\nRIAAECJAAAgRIACEWEgIjEj57rt1u/Gya55AAAgRIACECBAAQgQIACECBIAQAQJAiAABIESAABAi\nQAAIESAAhAgQAEIECAAh7dxM0U5l0Dncz3gCASBGgAAQIkAACBEgAIQIEABCBAgAIQIEgBABAkCI\nAAEgRIAAECJAAAgRIACECBAAQgQIACECBIAQAQJAiAABIESAABAiQAAIESAAhAgQAEIECAAhAgSA\nkP8B03ct/iu7u0oAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f883fe97710>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure(figsize=(8, 2.5 * n_digits))\n",
    "for iteration in range(n_digits):\n",
    "    plt.subplot(n_digits, 2, 1 + 2 * iteration)\n",
    "    plot_image(X_test[iteration])\n",
    "    plt.subplot(n_digits, 2, 2 + 2 * iteration)\n",
    "    plot_image(outputs_val[iteration])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Generate digits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "n_rows = 6\n",
    "n_cols = 10\n",
    "n_digits = n_rows * n_cols\n",
    "codings_rnd = np.random.normal(size=[n_digits, n_hidden3])\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_variational.ckpt\")\n",
    "    outputs_val = outputs.eval(feed_dict={codings: codings_rnd})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure generated_digits_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEGCAYAAAAwpAFeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvXl4lNXZ+P+ZzEwm+74nJGFN2Pdg2AUFocqixa217taq\n+G2ta3F5tdrXYmu1xaXWKhVxw4LyCiogyr4EISyBkIQkZN/3zD7P8/tjfuc4YbFCZkKwz+e6elUj\nZM48zznn3u9bp6oqGhoaGhoavQ2/C70ADQ0NDQ2NM6EJKA0NDQ2NXokmoDQ0NDQ0eiWagNLQ0NDQ\n6JVoAkpDQ0NDo1eiCSgNDQ0NjV6JJqA0NDQ0NHolmoDS0NDQ0OiVaAJKQ0NDQ6NXogkoDQ0NDY1e\niSagNDQ0NDR6JYYLvQAPtKaAGhoaGj8+dOf7FzULSkNDQ0OjV6IJKA0NDQ2NXokmoDQ0NDQ0eiWa\ngNLQ0NDQ6JX0piQJDQ0NDY1ehKqqiKG2Op2uy//3BJqA0rhoUFUVm81GTU0NR48eBSAoKIgJEyZg\nMpnw8+t5h4CiKDidThRFAcDf3/+CrEPjzCiKwt69ewHYuHEjBoOBO++8k6ioKIAefVfione5XAA4\nnU4cDgft7e1YrVYATCYTwcHB+Pv7YzKZ5Bp9KRTE3rXb7dTX19PS0sL+/fsBWLVqFc3NzdTU1Mhn\nNm3aNH71q1/Rt29fnz8/7SRpaGhoaPRKNAuqhxFalKqqWK1W9uzZw7vvvgvA/v37iYyMZOjQofzu\nd78DID4+Hp1Od94alMPhwGaz4efnh9FoBMBgMPSomX6+qKoq1w/Q3t7OiRMn2LhxIxs2bACgurqa\nu+++mwceeEBqnL5EURTMZjMAH374If/4xz+ora0lLi4OgBdeeIGJEydiMHTvaKmq+r3vyFMbFxqw\nzWbD4XCg1+vx9/cHwGg0otfru/W+VVXF6XSe9tmeaxR7SqfTyf8uuJB7TVEUdu/eDcBnn32GyWTi\nkksuITs7G/ju+fQUqqpKC8pisWCxWNi9ezfbtm0DoKmpiYSEBGbNmsXo0aMBCAsL6/Y7PNtaVFWl\ns7MTgG+//Za1a9dy8uRJ+cxaWlpwOBxybQD9+vWjo6MDh8Ph8zP3oxRQngekN13ELpdLboacnBwe\nffRRjh07Jjesy+XCZDJRU1PD1KlTAZg/f74ULOf6WQAnTpzgxRdfpLS0lL59+wKQnZ3N+PHjycjI\n6PZF6iscDgdlZWXs3btXXsCDBw8mICCgy6UyaNAgJk+e3CPfw+l0UlVVxRNPPAHAF198QUtLCy6X\ni5qaGgBWrlwpL7/u4rmPxT+LZ2G1WsnLyyMvL499+/YBUFpaSkdHB8HBwWRmZgIwY8YMZs2ahb+/\n/3mfBafTidVqpbKyUq7hwIEDHD58WD73sLAwkpOTiY6OJiwsDACz2UyfPn0ICQmRPwsICMDPz8/n\nbivPtX/wwQcAHDlyhOjoaI4dO0ZiYqJcd0xMDEaj0SfrOfUdKooifyY+c9SoUXR0dACwZs0aCgoK\niIqKIiIiAnDv8aCgIK8+M6F01NbWsnr1agB27dpFSUkJdXV18v4QSoifnx+hoaHy74eHh/eIYO+d\nt9N/wPOlK4qCy+VCp9PJh2qz2WhtbSU8PJyQkBCAHtWSTkVVVex2O7m5ubz33nsAfPzxx9TX16Oq\nqtR2dTodiqJgsVhISkoCzs/acTgc8tJavHgxzc3N8hCCWyv6/PPPaWtrIy0tDXBbahfyGQmEAF+2\nbBm7d+9m7NixXHXVVQD07dsXvV5Pfn6+PCzDhg1j7NixPlm7Z4DY6XRy5MgR/vjHP7J161bArQH7\n+/vLfQdw8OBBLBbLeSkVnnhaIoqiYLPZKC8vp7y8HIBNmzZRXFxMUVGRfBZ2u52EhAQADh06BEBd\nXR12u505c+YQEBAgf7fn//8n6uvrqampYeXKlQA0Nzdz9OhRqqqqpHbt7++PXq8nJCRECi2TyST3\nfmBgIAD33nsvM2fOJCYmRsYvfCmoOjo6OH78OOC+F4xGI83NzeTm5gJQVlZGdnY2EyZMkOfQW3ju\nHyGcPC1enU6H0WgkPDycfv36AZCcnIzVaqWjo4Ndu3YB7r03bNgwKdzF3z0fxGc7HA4aGhr4y1/+\nwhdffCH/m8lkIioqSr4vYZWbTCbGjx8PQFZWlrxXT02g8DYXnYASL1q4ferr66mrqyMwMJCWlhYA\nioqKqKysJCEhgYULFwIQERHR48Fr4RZpbm5m7dq1rFy5kpKSEsCtPYWFhaGqqrw4XC4XgYGBXH31\n1QwdOhQ4vxff2trK008/Dbifz9ChQ3nmmWcYPHgw4H6GTU1NbNu2jU8++QSAKVOmMG3aNIKDgy9Y\nkN9ms7Fo0SIAdu7cSXZ2Nj/72c9ISUkB3MHi9vZ2vvjiC1pbWwG3u0E8P28hLhaRAAHQ0NDApk2b\naG5uJj09HYC4uDhMJhN79+6V7o/W1lbq6uoIDQ3ttltNCL729naeffZZjh8/TkNDAwChoaEYDAbG\njRsnL44xY8YQFhZGa2sre/bsAdxW1YEDBxgxYoRc97m6i6Kioujs7CQ+Ph6A2tpaWltbcTqd8lIP\nDg7GYDBQV1dHe3u7/A5CUxdCa8mSJdTW1nLHHXcQFBQk1+MrOjo6pNID7v0ybdo0eVHv3buX4uJi\nhg8fLpUKb1y2Yv+Iz2lvb6etrY3q6mqCg4MBCAkJISAggKqqKjZt2gTAgQMHsFgsHD16VCYC1dTU\noCgKQ4cOPe2ZnctahbIgvvfzzz9Pbm4uFosFgISEBAIDA9Hr9TQ3NwNuQaaqKuHh4QwfPhyAmTNn\nSgHma7QkCQ0NDQ2NXkmvt6A8A7GeLg+RlllVVcXRo0eJiYmRgeo+ffpw+PBh8vLypMZw8803S+2j\np9ZdV1cHwLp169i8eTPBwcGMGTMGcFtLJ06cICQkhD59+gCQnp7OrFmzuOSSS6SGcq7anMvlIi8v\nT7qCEhISePPNN0lMTOzyHAMCApg+fTpHjhwB4L333uPf//43t912m9SUwsLCesyaUlWVt99+m6+/\n/hpwa5fPPPMMqampcg2KopCTk8P+/fsZNGgQANddd53X1niqO0ZovwCBgYFMmzaN8ePHy30UFhZG\ndXU1VVVV0nrv7OyU1n13UBRFxnxefPFFPv74YwwGA5GRkYDbEpg4cSKXXXaZjKeYTCYMBgMOh0PG\nG7/44gva2tpobm7u8uf8/Px+sOViNBqJjY1l7NixAPTv35+0tDScTifJycmAO7akKAorVqyQ7kWr\n1Sqfo3iWzc3N7Nu3j5///OfSkvAl+fn50hLV6XTcdNNNjB49WlrGdrsdf39/r7v3XC4XFouFxsZG\nwB1z7uzsJD8/X3oyOjs7OXHiBJ9++qm0WGw2GzExMaSlpdG/f38Ahg4dSnJyMkaj8YwJKj8UVVXl\nvfD0009z8OBB7HZ7lzii0+mksrKS2tpa+XeCgoIYO3YsCxYsANwxsZ6KXfd6AeX5EoRfXlEUabY3\nNjYSExPT5aHl5ORgs9koKSmRLz4+Pp4rr7zS6xvxbDgcDrZs2QLA6tWrGThwIKNHj5Zm+6FDh7DZ\nbAwdOlS++CuvvJLg4OBuuRhcLhdtbW0yFjFy5EiZCSjQ6XQYDAYSEhJ49NFHAfdBfvfdd3n11VcZ\nOHAg4I4XxMTE+Dw2paoqtbW1PPbYY/JnN9xwAyNHjsTPz09ebg0NDSxduhSDwcAdd9wBIAPJ3loH\nuIWDiK2I7x4QEMDAgQMxGAxyD5nNZvbs2UNFRYV0k5hMJvnP3VlHRUUFf/nLXwC3kNHr9aSnpzN9\n+nQAMjIymDRpEjExMV3qZcAdDxJCMiYmhgMHDhAVFSXdxud6ufn5+RESEsKECRMA9yU6evToLvFR\nu93O+vXrqaqqkkqhy+WSnyXOZnR0NHfccQcRERE9kiTx73//u8v3EEJavN/k5GQMBgMmk8mrCQgO\nh4MTJ07IGqzjx48zduxYRo4cSUFBAQBvv/02dXV1OBwO+XwiIyO57bbbWLhwoVQyRHxPJCucL21t\nbbz++uuAOwwi9qnYFykpKZSVldHU1CSfj8j+nT17NgMGDADcZ6Gnks96vYA6G+IAFhcXoygKHR0d\nlJWVAe4kgICAABwOhxRQ77zzDh0dHVx11VUyqOyL1E1Apm4eOHAAcMcLUlNT2bFjhwzYBgQEMG7c\nOP7nf/5HvnhvWAJ2u52ysjKKiooAGDt27Fm/o16vl9lVI0aM4Be/+AUvvviiTHkFeOCBBwgPD/fp\nhrTZbMyfP5+Ojg4Zb3rqqadkTECkdT/66KNUVFRw6aWXMmXKFMC7hZZCQFmtVpxOJ0ajUV4cBoNB\nZp8JCgsLefnll6mtrZVCVK/XEx4e3u11lJSUUFVVBbiTMa666iruvPNOGUwX1tKpmV0i7lNaWgq4\nCy2Li4tpampi9uzZAOccsxMXo3gfLpeL5uZmtm7dSl5eHgAnT57k22+/pb29XT4LIZyMRqO8BD3T\n8H19yamqyqZNm7pYHbt372bo0KEya85qtZKenu7VtaiqysmTJ3nuuefk85k8ebJUFL/88kvAfU85\nnU5CQ0OZO3cuAL/73e/o27dvl3d0qgfpfNd0/Phx/u///g9wx6btdjuRkZEy4zMmJobS0tIuHgCR\nlj979uzTvDo9IaQuOgEltO0VK1YA7iylESNGUFBQILXpiRMn4u/vz7x586R5/+2337JixQpeeukl\nWV/w5JNPkpKS4nULweFwsHbtWqmFjBw5kpqaGqqqqqRwvOuuu7jhhhu87nYMDAyUAVlwp5nb7faz\nXkpik5lMJvr3709iYiLHjh0DYNu2bSxatIjQ0FCfZFyJd3PPPfdw4MABjEYjy5cvB5BV6zabjVWr\nVgGwefNmoqOjmTBhgk+CtMLtU15eTnV1NX379iU2NhZwCx5hWeXk5Mh1l5aWoiiKfL7XXXcdffr0\n6fZzqqyslBf91KlT+e1vf0tqaup/DI7rdDosFgt/+MMfADh8+DAjR47ksssuk3/3fIS6pxVksVjY\ntWsXn3zyCYcPHwbcyQietVLgfmaRkZEsXryYxYsXA25lzVeK4akoikJ7e3sXy/jLL78kNjaWcePG\nAW6rwXN/ewO73c7NN9/M8ePH5TNvbm6WCSQi0cVgMBAdHc3//u//ct111wFIS+5Mz6e7STf19fXy\nHTmdztMEjXDpe77rxMREbrzxRuLi4nok6/JUtCQJDQ0NDY1eyUVnQdntdvbt20dhYSHglvD+/v7M\nnTuXjIwMwO3CEC4EoT2JWof7779fut7eeustbr75ZhITE0/z458vqqrS0dHBli1bOHnyJOC2BqxW\nK0lJSVKTnD59uk8SEPz8/EhLS5O/++jRo5SUlDBw4MAzat+e7g9wJycI90dmZibR0dFn/PPd1aJU\nVZXv8MMPP0RRFG699VYmT54s/7vNZmPNmjW89NJLgFvrEwH+7rg7zoawWFpaWvjkk08ICwtj4sSJ\ngDuGWVlZyaeffioLG61WK0ajkeDgYK6//noAnn32Wa9U12dnZ0tLbdCgQURERPwgy8PhcPDb3/6W\nzZs3AxAbG8vMmTOZO3duF6vzXN+fZ/wjMjKSefPm0dnZKeMpIrUcvjtDcXFx0joQcbueLGHQ6XSM\nGDFC1q2B27VVWFjIrFmzAHdMzFsBf7Enm5qaqK2tlR1cwJ3M9dVXX9HQ0CDdfi6Xi8zMTKZOnerz\nOK9Op2P48OHSeyRcfBaLRaa4x8fHU1JSgqqqcg8PGjSI7OzsC1YjeVEJKFEbkpeXJ/3806ZNY/r0\n6f+xfkev15Odnc3q1atZs2YN4K6GPnToEP7+/jI7KigoqNumdHt7O9XV1eTn5wNul4hOp+Ouu+7y\nSezEE51Ox4IFC3jllVcAd6ZQWVkZ4eHh8pmJzecZLLfZbBQUFHDixAm5GWNjYwkKCjpNGHhj7Waz\nmQcffFCuefz48fz+97+XQsJisbB8+XLeeustmXkUFhZGUlISQ4cOlZeKN90N4neGh4djNpspLi6W\nLqyamhqZ3SSeR1BQEEOHDuXuu+8+zUXTHXQ6HbGxsTLms3//fmbOnEloaOj3FgCrqsqTTz7JypUr\npUB44okn+OlPf+qVTgni7/v7+5OcnCwTVQDef/99CgoKsNvt8s9FR0fLIuoL0dHFz8+Pl19+mUsv\nvRRw7zmr1YrZbCY6OhrAJ9lowcHBTJo0iU8//VTu54KCAmpra3G5XNTX18s/29jYyKeffirXGBgY\nSL9+/bpd6H0qOp2OlJQUXnjhBcB9L+Tk5HSJ0zc1NcnsaOGyjoqKkjHgC8FFJ6BaW1sJCAiQqdBT\np04lJCTktEtTBIs9f67X6+nTpw9XXHEFALm5ufz73/+msLBQthYaO3Zstzat8N9arVaZZu5yuTAY\nDBQWFvaIBhkQEMBHH30EwJYtWwgLC6OhoUFmPgpfs8PhkBuxoqKCr7/+mra2Nvn9RfGipyDz1kVT\nUVEh06gjIiKYOXMmtbW1Muvp73//O3v37sVms8k40KBBg7j88ssZMmSITy4W8d1E7CIvL08eXqfT\nKTViEetcsGABt9xyCyNHjpRC3xvPR6fTERAQwPz58wFYu3Ytu3btwmg0ypKEM11g+/fvZ/ny5RiN\nRn75y18CsGjRIp9cdnq9nujoaB544AEAJk2axH333UdeXp4U4HV1dXz99dekpaXJWGtPtTgS9OnT\nh8suuwxwZ0O2t7dz6NAhuR5vrkX8rpCQEK6++mpycnJkvMnf35/Ozk7sdru8AwIDA4mLi2P//v18\n/vnngFuITpo0iSVLlsh4tbfWKDJBwV1WcujQIV566SW++uorwK0UirMuEiUOHjzIwYMHSU1N7RIv\n17L4zoJer8dgMMhU6DP1GDu1xYjnz51Op7x0ioqKZBsRcQl6Vr2fDzqdjpiYGDIzM9mxYwfgPpTB\nwcEUFBTIuqMRI0b47CXrdDqZZn7ttdficDi6uDvFJnS5XDJRISoqirS0NNatWycvtGHDhmE0Grtc\nKt5Ys6Io0r0H7pRpk8nE8uXLWb9+PfBdkkB0dLQ8VI888gjjxo3zSYNKl8sla1Y2btxIQUEBjY2N\nMtEF3BdKenq67NywcOFCRowYQWBgoNffpcFgkEpYREQEZWVlfPvtt7LGaPr06YSHh6Moimxr9dRT\nTxEZGcmf//xnZsyYIX+PLxF7ZdSoUYwbN47CwkIZiO/s7OT1118nJiZGComIiAifWL9nIyAggEsu\nuQSADRs20NHRgdls9uln6/V65s+fT2BgoFQUS0pKaGhokKUlAPPmzWPixIkcPHhQComcnBxWrVrF\nli1bpCs5ISHBa4qtp5t2/PjxTJw4UWbtdnZ2yrtBlArU19ezcuVKUlJSZA3n+dZontd6ff4JGhoa\nGhoa58FFZUGpqorBYGDatGnSXdXW1nZay3yXy4XD4ZB9pMAdPBbDy0TVf3p6OrNnz2bUqFFSE/SG\npiJSOkUcICoqiiuvvJLt27dz//33A+4iPVHT4gtOdW2eaY3wXWKAwWCgpqaGwsJC6cJKSkpCr9d7\nvdbI4XDQ0dEhXQaRkZE0NDTQ0dEhXQuiOHT8+PHSGrjkkku87q4C994oKipi6dKlgFvTFvtLdDsI\nDQ1l2rRpZGVlyfcaGRnZrS7h34dOp5MungEDBhAcHMyXX37JZ599BrityYiICJqbm2UfwNGjR/Ph\nhx8ydOjQHu+nqCgKRqOR6OhoqX3b7XZUVSUnJ0d2RUhPTycsLKxLwbwvNXE/Pz/5Lm02G4qi0L9/\nf58H/Q0GA5dddpk8S6+88op0ZYt+k7Gxseh0OgYMGCDroL7++muWLl1KcXGxvCuWLVt2WrG9t9bY\n2traJcYkUtyFlavX6ykpKWH16tVyP2ZmZvbIaBu4yAQUuF16DodDBs5LS0tJTU0lKSlJurUaGhoo\nKioiNzdXZheJzWkymWQGz5gxY7rdueFMFBcXk5OTI+M7jz32GDfeeCP5+fn86U9/AtwFi8uWLbtg\n2THiO4vPNxgMHD58GIvFQmpqKuCbVkeqqmKxWAgMDJSXVlhYGIMHD6aqqkquKyAggNTUVKZNm8ZN\nN90EnDnu4o31VFRUcNNNN8mMNIfDgdFoJCUlRWbnXX755fTt2xedTndasaOvik7FszeZTCQlJTF5\n8mRZJ1ZSUiJrWUSCz+LFixk8ePAZ47Fwevamt5qiit89Z84c2tvb5e91Op0EBQURFhYm3ZDFxcXM\nmTNHFhmfui5vIpKqxB53OBz4+fkRFRXVI+4po9Eo3dNBQUE4nU4mTZokW7KJden1erm3582bR1ZW\nFvfff78MB7z++us8/vjjXnfXtrW18fHHH0uXrIgtxsTESOXRarXKhArRLHjAgAE+U8xO5aISUHq9\nnqCgIIYMGSLbvZeXl7Nx40bCw8OZM2cO4BZax44d63LhhYWF0adPH+bNmyez2XzR9sjlcrF69Wrs\ndrtsqyL6jo0cOVIGubdt20ZHR0e3uw54C51OR3FxMYGBgTIV1RexFXEIRo4cKUd9BAcH09TURE1N\njWy/otfrGThwINdee63Xu5V7YrFYePTRRzl69Kg8qKGhoUydOpXnnntOCmuRndfR0SE7PDQ2NjJw\n4EAGDBjgU0VDp9Ph7+9Penq67Dpy8OBBXC4XoaGh3HfffYA7YUinc49sOfW9ec4h0uv12O32bmnB\nIs4rYpjCQ5GQkCCfWWxsLK2trV1GRzQ0NKDT6bj88su7JAF4o2zB8zvCd96BiooK+e86nU6eS1/j\nmZovRpQUFxczatSo7/07CQkJPP3009x9992AO0nm3nvvlXFyb+ByuVi1apW0vsGtFMbHxzNjxgyZ\nvXrixAmcTqcszga46qqryMzM9JqS831cVAIKkDNnRM1T//79ycrKoqWlRZryRqMRp9PJ8OHDZXae\nSJn2dYsVu93Op59+isVikRe9mIvT2Ngo59C0trbKmVW9AdE7DJCWqK8C7KK/XkxMjPz3wsJC1q1b\nJ4VERkYGTz31lM+0XXGRVVVVceDAgS6jI7Kysli6dCkpKSldrBFxCYokm9raWjIzM+nXr5/PLWFF\nUaiurpafM2bMGBoaGrj66qtlirvRaJRJHZ6JCKdaUKIjRncFlN1ul89i48aNlJWV0dnZKdOoMzIy\ncDgcfPPNN3L8h81mY9OmTQwcOFBa0CILtzvvWVEUWltbZYunyMhIoqOjcTgcVFdXA+7vHxwcLF1s\nPYFQrvR6vaw5Ei2nzpR9LIiPj5fu5by8PFasWMGvf/1rr3k0zGYzK1eupKWlpct8KpfLxdatW7t4\nnoSl7lnf5g2l4oegJUloaGhoaPRKLjoLCrr2BROdiEWreHCnsiYlJZGent7j9Re1tbVUVVVhMBhk\nEkRHRwdNTU3cd9997N+/H3CPL/emyd5djh8/TlNTU5fgvOe4CW8hhk2azWZZI7JlyxY+//xzSktL\npQtr6dKlpKWl+eydie/W2dlJXFwcDQ0NsnHutGnT0Ol0MpkG3DEEl8vF3r17ZWzAYDB4tZv62RBx\nu3379skuH5GRkVx22WXMnTtX7nGn04nT6ewyBNNoNMqJ0+LMqKra7VEXwpL88MMPAfjqq69obW3F\n4XDImNiJEyfIz8+nurpaJk6IkSnDhw+X3gPPUeLng7AQOzo6ePfddwF3icTo0aNpbm6WvSVFw1sR\nS+kJREr2lClTKC4uJj8/nyeeeAKAO++8k4EDB55WRG232zl8+LB0s7lcLmpqanC5XF7pdANuz8Gx\nY8e6uEVtNpu0fj3dwYGBgQwZMoQnn3wScO+9nkrCuSgF1KmIuIZ4aAkJCSQmJvZoBbvnIRGNRUWT\n07y8PPbv309tba1sgvq3v/3Np7GVH4q4qNeuXStnv2RlZQG+cfGJIsDi4mJZI7J161ba2tpISkpi\n2bJlgNuF1RPvTlVVxowZQ2Njoxxv4HA42LBhA6qqyqa76enp1NTU0NnZKWMsEydOZPTo0T4/rKJB\n8rp162QiR3h4ONHR0VIAwXcjzT2TScT79fPzkwqcN9zc4rIXAjMvL4+Ghgb0en2XprFCWAr3qV6v\nZ/DgwWRnZ0vh7o0OF35+fsTExHD11VcD7onMb7zxBuXl5dL1HxQUxLhx47otEM8F8b1vueUWmpub\nWb16tczEPHbsGCkpKUycOFHWGIWGhvLll1/ywQcfyHVHRUWRmZkpk3e8RWBgoJxMDUhFRq/XSwUm\nKyuL66+/np/85CdS8ejJDNEfhYCCroHWnvKPeiIEVHl5OUajEavVSnFxMeAuCDYYDAwePJhXX30V\ncKcEX4j2L6ciNNtDhw5hMBgIDw+XMShfpU8bDAY++ugjdu7cCbhbrAwcOJB3331XDnPrqWeTkZHB\nL3/5S5KSkuRnRkZG0tbW1iVFuaqqioiICPr27Sv78w0ePJjAwMAeObBmsxl/f395QcXHxxMXF0dE\nRIS0/IxGo2zV5Wn9emOW0KmIeI7wEkRHR9Pc3CxjFp5/xt/fX7YWGj16NLfffjuZmZle7eYgum+I\noty+ffvy/vvv09zcLEe4mM1mHn/8cZ9kg37fusCdCPTwww/zs5/9THaNOHLkCHa7nePHj8v7o7m5\nmTVr1lBXV8dPfvITwP1sZ82a5dUO/klJSTz22GO88MILUkjp9XpSU1OZPXs299xzD+DeZz2VsXcm\ndL5ounme9JqFdIeysjKWLl3KZ599Jg/q2LFjWbhwIQsXLpTaW0/XqZwJTwvhvvvuY9u2bcydO5fn\nn38ecGtzvtiYLpeL4uJi2TRTr9czY8aMbvdBPBdO7TTidDrlpd7R0SF7ynkGkAMCAk6zBk5tp+UL\nxHTdV155RSbZJCcnc/fdd5OWliYvLpPJJC9fz5lMAlFjFhAQ4JWkDjEcE2DPnj2sWrWKL774Qn5m\nRkYGQ4YMYc6cOVKQ2e12Bg8e3MWKE8/P21lhItNQnEO9Xn/Byjo81yRQFAWXy9Vl4rDo5Sm6qMB3\ns8h6g0J7npz3wi/8LamhoaGhoXEGNAvqvxhP7XLPnj289NJLzJ49m1tuuQXwTWGsxvlht9tpb2+X\nLse4uDgxvKIBAAAgAElEQVQZXBeatbBAzlSQ63Q6Zc1LdHS0VywJz56X4LYIrFar/N3Cujy1l+Op\nk4k1fvSctwWlCaj/cjxdXZ5tjzQ0NDS8hCagNDQ0NDR6JVoMSkNDQ0Pjx4UmoDQ0NDQ0eiWagNLQ\n0NDQ6JVoAkpDQ0NDo1eiCagLhKqqlJSUyDY1GhoXOyLtXPxPDA11Op3YbDZZKKyh8UPRsvh6EFEn\nAu65OK2trbS0tDB27FjAN/OXLlZEE8uLvILep3h2SAB6pKvF2RCzoaxWq1S6ampqCA4OJiYmRo72\n0N7lxYWiKN7YU1oWn4aGhobGj4sfRUWmsAI9uzrX1tZSX1/P3r17AbjkkkvIzMwkODj4gmiZYmDh\ns88+C8C+ffvo168fGRkZsjB23Lhx//XdG4RV0NraSn19PREREXI8gqZ9f4dwo9lsNrn/TSaTT6ZE\n/xBEE+Dg4GDpJQgLC6OgoICYmJgzjp3X6J14dgixWq0X1LNz0Qso0a6nra2NTZs2AbBhwwZsNhsl\nJSVyPEFoaCjjx4/nxRdfJCkpCei5wyIaQBYXF8vO02azmW3bttHc3MzcuXPln/tvRlVVebktX76c\nd999l9TUVP75z38C7hY9PTWCQ1z+69evB+Dzzz+nuLiYpqYm4uLiABg0aBDjxo1j4sSJp42G9+Xa\nwK2EFRYWcvLkSTlXKTk5maSkJJ+v4fvQ6XRSyTKZTAQEBGA2m7s1vfd8EJ1RHA6HnOvV3t5OVVUV\neXl5ctaSxWLBbrdzzTXXyHNoMpl6TIkVe82zFdSFRMQOq6qqADh8+DCTJk0iMjLygqztohVQoi2P\n3W6nqamJ0tJSOc/n7rvvxmKxsHnzZjmSuqWlhYaGBjo7O3tcmxNjr3Nzc+Uaf/3rX1NVVUVycrIc\nfd6TG8AzmA3uwX1FRUVs3ryZ48ePA9CvXz9mzJjByJEjeyyGIARUdXU1NTU1tLe3yx5y4jn5GjHO\nvLS0lPfeew+AHTt20NzcLGfmgHvQ4ttvv016ejp33nknALfffrvPrHRFUWT38Pfee4933nkHq9Uq\n301WVha33norw4cP77Yl7jlH6lzw7O9YUlJCfn4+UVFRcpaQrxHrdjgc1NbWsmzZMr744gvAPUy0\no6NDxsvA/f0CAwMpLCwkPT0dgJEjR/aIgLJarTz55JNs2bKF22+/HYDbbrvtgrUaE/u+srKSF198\nEXAr+4sXL+bee+/VBNQPRTxIQE7xHDJkiBw7IFr5Z2VlMXnyZAD+9a9/0b9/f+Li4np8pIPFYsFm\ns5Gdnc3o0aOB70YeWCwWLBYL4HZR+srF59lrz2q1Ultby44dO+QFvG/fPtra2roMmzMYDPzpT3/i\nj3/8o2wg68vDo6qqvGwTExPlvCAhtHoKnU6H2WzmhRde4JtvvgHc++zUER2KouB0OikoKODxxx8H\noKCggOeff56QkBCvXXLi3bW0tPCPf/wDgGXLltHY2IjT6ZT7+ejRo+zbt49nnnmG6dOnA+ff8Pd8\n164oCmVlZQB8+OGHHD9+nISEBPr37w/4VsERVi/A119/zRtvvMGOHTvkUEWXyyUTSTxHptjtdo4e\nPcprr70GIGe2+QpxDpcvX85rr72Gqqrk5+fL/+btsSP/CSGsLRYLNTU1fPDBB2zZsgWA+vr6LvOq\nehotSUJDQ0NDo1dy0VpQIrU2JCSEoKAggoKCpNYntA+DwSCnRbpcLpKSknos4OcZTzl48CDBwcEM\nHjxYjrkW6/e0SISm7Cv3kHDdvfPOO6xfv56KigqpXZ7q8gP3M7PZbKxevZrrr78ecD9vXyGsBHDH\nAfr160dhYaF8hz2pWbpcLhobG3E4HPJnfn5+mEwmaamLhACz2Sw1948++oiQkBCWLFki443eWLPd\nbue1115j2bJlADQ2NnaZmgtuDfjYsWO8+OKLMk42bNiwHhvSJwZgPv300wCsX78ek8nEjTfeeN4u\nw3P9fLF/9+3bJzV/zz2bnJzM7NmzufLKKwGorKzk5Zdf5uDBg3LSbVVVFWlpaT7ba8JjsmrVKiwW\nCyaTSb6jniyrUBSFzs5OSktLATh+/DhFRUWUlJRIT4rT6bygsfGLUkB5vkCbzSaDmqfOwcnPz+eJ\nJ54A3KOLx48f32NZcg6HQ04/zc3NZeLEicTFxcmN6BkUFYfXV0W7qqrS0NDAc889B8CmTZuw2WxY\nLJYuB8NkMnURrGLKbHh4uM998p5xAYDY2Fh5eC9EjK6xsZHy8nL5fAICAujXrx/z58/n8ssvByA1\nNZWGhgaWL1/OqlWrAPd+fPPNN0lKSuK+++4Duu8WVRSFb775htdee436+nrgO2Ht5+cnf78oiD14\n8CAvv/wyAM8++yyJiYk+f3ZCuVi8eDGffvop4H4WVquVf/7zn4waNQqA/v37+8xN7DnWPjIyksTE\nRKZPny7d6jNmzCA5ORmj0Sj/nNPp5MiRI+Tl5cl9v2PHDtLS0nyyRs8p1kVFRfLnIlO1J84ZuN/N\n7t272bRpE3V1dQCMGjWKqVOnMnv2bAoLCwEoLS3l2LFjPl3T93FRCij47rIyGo1S6HjGCCorK1m0\naJHcdLfddhvDhw/vkeCnoihUVVXJgxofH8+gQYPkgDlPhPAASEpK8slFoqpqF03JbDZjNBpJTEwk\nISEBcAuEtLQ0ioqKKC4uBqCuro6oqCgWLVrUY1lY4vsfP36cqqoqIiIieizALlBVlYCAAIYMGUJl\nZSUA4eHhzJ07l7vuuquLwIyJiWHRokUy26+lpQVVVcnNzZUJOjExMd16r1arlVdffZX6+vouCT4m\nk4nIyEgZ36mqqpIa744dOwC3MnL99df7LP1cKFfV1dXccsstbN++XVqd4qwdOHBAJgGMGTOGBx98\nkOTkZK9bdiLVHeCGG27glltuOat14mkVtLS0yPgUuOOIvhTowoLS6XTyM2fPni3X6Cs875p33nmH\n999/H5PJxD333APArFmzCAkJwWKxkJiYKP9eWFiYlmZ+Luh0OrnpjEYjZrMZnU4nD8a3337Ln//8\nZ0JCQqQF9dOf/rTHXB0Oh4NVq1aRl5cHwNy5c8/oWhQmtnBhiVRlbyNSf4XbJyYmhr59+zJx4kR5\n+YeHh1NbW0txcbEURhkZGSxZsoQrrriiR56d0+mUB6iurg6r1Upqaqp0lfUUOp0Of39//Pz8iIqK\nAtwp7klJSfj7+0tLTyhCn3zyibyoAwMDURSFjo4OeRGdr2tSXF5lZWUcOnQIRVHk7wkICCAtLY0H\nH3yQGTNmAG6rb9u2bezYsUNq59u3b2fOnDndFpJnQlEUDhw4AMC1115LWVkZiqLI/ZOUlERqairN\nzc3S8lu9ejW5ubk89dRTTJo0SX4XbyG+Y2Rk5FndZZ7WekNDA8ePHycwMFCue+bMmV5bz5kQz0Io\nMwApKSk+/Uxwny+hSK1Zs4bw8HAWL17MFVdcASA9KOKfwf1uhAJ0IdCSJDQ0NDQ0eiUXpQXliU6n\no6qqivfee4+TJ08CbhM9ICCA22+/nauvvhrouTHmiqKwZ88ePvroI0JDQwFIS0s7zQIRKbGHDh2S\niRNncgF6A51OR1RUFGPGjAHcwf2xY8cyfPjwLq6OkJAQCgsL6dOnDwAPP/wwqampPRK3E27IrVu3\nAm4r2GKxEBsb22OWrydWqxWj0ShdoOnp6Wzfvp38/HyZMJKbm8vnn3+OzWbrEvtITk4mOztbpsmf\nr9tGaPnbtm2jtbUVnU4nrY2+ffvy8MMPc9VVV8mkjaSkJIKDgzlx4oSMIRQWFnL06FEmTZrk1TPg\ndDo5dOgQ1113HYC0ngIDA1m4cCHg3j/+/v5s2LBBFtEXFhZSVlbGI488wm9+8xsArrvuOq+tzfP8\nnOrOUxQFRVFkAgzA2rVrSUpKoq2tTVq8sbGxPkvI0el0MqHGYrGgqioRERE+TT4C9zMwm83s2bMH\ncFtsd911FxMnTuzivlcUhdbWVnnmRo8ezSWXXKK5+M4VzxhUWFgYwcHB0sUXFhZGbGwsI0eO7OIP\nF/VRvkAcgsbGRv76179iNpu59dZbgTPHlhRFYffu3bz77rvcddddco2+xLN6vrOzE39/f2JjY4Hv\nkk3mzJnDpZdeCrhdgT3ZFqqjo4ONGzcCUF5ejsvl6lKX1VOoqipdncIXn5ubi9lsxs/Pj507dwLu\nvdfa2kpiYiKLFi0CvnPVTJkyRQoop9N5XhewuDArKioICgrCaDQyf/58AO6//3769u1LYGBgl4C/\nv78/dXV1VFdXA+7i1E2bNpGVlXVags754nQ6qa6u5te//rWseVJVlbCwMK644gr+8Ic/AG63qNPp\nZOLEifK7BAYGsmvXLsrLy+VznD9/PsHBwd1el6Iosj5SURT0ej0Oh0PGAjs6OmhoaOgS9K+vr2fA\ngAFERERgNpsB359DsS+EEBwzZozPFWin08nx48dlQkRWVhbjxo3DYDBI97SqqnR0dHDs2DHp2hYx\nfJfLdUEKiC9aASXw8/MjOTmZBx54gJKSEgDefvttmpubycnJkTGNhIQEhgwZQlBQkE+ElDgYK1eu\npLy8nBEjRvDzn/8ccKeUn6rNNTQ08Oqrr2IwGHqs15woBqyrq8PpdBIRESEthLa2NpKSkrj00ktl\nEkBPCidVVamoqJAHqKmpCZfL1SXNPCoqqsuz9BVOp5MVK1awd+9eOjs7Aff7DQgIICgoiOjoaMCd\nkZaRkcGsWbOkoDcYDHR2dhIZGSktz/M52J7ZlGazmfDwcNLS0vjFL34BwIABA05raeRyufjqq6/Y\nsWOHvGxNJhMVFRU0NjbKd32+gsqzP9ubb77JsWPH5O9KSEjgF7/4Bbfffrv8WXt7OydOnMBsNsu0\n7mHDhtHW1kZ1dTXt7e1d1tMdVFWlqalJCsfi4mICAwM5dOiQTBAZNmwYmZmZTJgwQVosWVlZBAcH\nU1paKpWj/Px8BgwY4DPLXbSm8vf3R1EU2V3GlyiKwqpVq2RZSXNzM5WVlURFRcl45Zdffsnhw4ex\nWq2y3KO2tpaSkhLS09OlB6YnPRoXvYCC74La4kVfffXV1NTUUF5ezieffAK4L+Xf/va3TJ482eua\ngKqq1NTUAO7sGHCntIoL3m63yxoNkTixZMkSioqKWLBgQZcstVNrDrx1Gfv5+cnU2fr6enJzc8nP\nz5faXExMDEOGDCErK+uCmPM2m429e/dSW1sLuBNN/Pz8CAkJkYK1f//+xMbG+swVKjTJyspK2tra\nsFqt8h1GRUUxffp05s+fz/jx4wG34BElDuLvtrW1ScHUHQHvcDhkFui6detoaGhg1KhR8nJzuVw4\nHA454gIgJyeHF198UbqvxBrr6+spLy+XrmRPq+tcEHvzyJEjrF+/HqfTKTXt3/zmN1x99dXodDp2\n794NuLP3zGYz8+fPJzk5GfguQaezs5Phw4cD3lGE6uvruffee/nqq68A9yWamJhIv379pMv61ltv\nlXVhnokuVquV6upqmeWakJCA0+n02UUslAeHw4Gqqj3iQhcJNqI3aXFxMfv27aOlpUWeuaamJtnN\nRXieXC4XdXV1XHvttV3aMSUkJPSIAqslSWhoaGho9Ep+FBYUuC0NoVUPHDiQ+Ph40tPTpRvBarVy\n4MABr/riBYqiSMvIarUSFRWFv7+//Nlnn31GXV0d+/fvl1pacHAwEydOZMaMGVKbs9vtGAwG9Hq9\n1y2pgIAAXn/9dQDuuOMO9u/fT3t7u3SfiQGKdXV1PV53BO7Els8//1y6IESN0X333cfIkSMBt1si\nPz+ffv36ERwcLP+cN/B0qW3atEkWUw4aNAiA++67j8svv/yMna49e0N2dnZSWFjI4MGDzzvwraoq\nZWVlsk9iZWUlgYGB+Pv7y+SHoqIiCgoKqK2tlXtKJCA4HA75XBwOB5WVlezfv79LKnNgYOA5Wwhi\nn5aUlOBwOEhLS5Nu7Ouuuw5VVTl69KhMdAkODmbu3LldzlxLSwvt7e0EBQVxySWXAN23oERiUmFh\noQz4L1q0iMWLF5OYmCiTSM6Udi4smJCQEOkK3Lp1KzfddJN023obkagg+u4Jt7avqaurk9ab2Wym\noaEBvV4vPUoxMTEEBQURHh7e5Rw2NjbS1tbG888/D8Bbb73Fgw8+yPXXXy9LQHzlev/RCCjPRpEH\nDhwgNDSU+Ph4mVFkMpnk4RAb1ls4nU4pCBMTEzEajRQVFckakc2bN9PU1CRNaLEem81GWVmZvGxF\nsNazZkq4GrrrbtDpdHLMyIoVK/jkk0/YtGkTR48eBdwV4ydOnOD555/njTfeAOix2UKqqnLo0CGq\nqqrkAQoICGDhwoVMnTpVukCcTidffPEFl112mXQPecM9oigKFouFffv2Ae4WPaLT/L333gvAFVdc\ncVbXosvlkhmkr7/+OqmpqfLyPd/1vPnmmxw6dAhwKz2qqlJQUMCKFSsAt4CqqqrCZrPJRBJx4Z36\nu8rKyvj6669lHdzkyZO7FLD+UESiw/bt2wkODmbMmDHceOONgHucTUlJCXl5eTJGN3v2bIYMGQIg\nz+bu3buxWq3Ex8d7rb5GZKclJSVx//33A+5C3YCAgP94aYri3v79+8us27y8PJ92dRHuYJGgcPjw\nYZ9PWDCZTEyZMkUKw5qaGhISEujTpw/Z2dmAu5NEaGgoer1eKg1ms5nNmzezc+dOeVc0NTXx9ttv\nExQUxKxZswC3C9wXY15+FAJKURRKS0tZs2YN4NYUbrnlFmJiYuRGS05O5vjx41LT9SY6nU4e/n79\n+uHn50dmZqbsQhAfHy/nqwjrJCUlhaamJnJycqQPuKOjg8zMTJKSkqRWoygKBoMBl8vlFSEFEBER\nwcyZM4mNjZUdB1asWEFLSwubN2+WVpWvNMhTsVqtbN26Vabdis9esGABBoNBCq2TJ08SFRXllYwv\nT8xmM2+88QYfffQR4O7IkJqaytKlS8nKygLOnugg9t4vf/lLwG3t3HTTTd2Kk7lcLtauXSuVHtET\n8fjx41Jo2e32Lt0PBMKTIC4Y0bk7LCxMXsBCqJ2LcFdVVWr+Bw8epKmpCbPZLPd1SUkJq1evprW1\nlQEDBgDuS9BisRAeHi6tqq1bt+JyubjqqqukIDvfmLC46I8fP05hYSHZ2dldik7P5fl7pvDbbDZO\nnjzpsxIHYZWbTCacTicdHR1dxn/4Aj8/PxYtWsSECRMAd6w0ODiYtLS0LhbmqfOpFEXh+uuvZ/78\n+WzevBmAf/zjHyiKws6dO2XbL19x0Qsom83Ga6+9xjfffCMtkSuvvJLExET8/PxkZt+GDRsoLS2V\nWVneRKfTSe0yOTmZgQMHEhcXJ0c1HDp0iJaWFgIDA6W2otfrSU1NZcSIEfL3BAQEdKnZEIj0YW8S\nEhLClClTGDp0KADHjh3jyy+/pK2tTbq3fC2gxHfNy8sjLy9Ppip7fnZTU5NUKkJDQ7n22msJDg72\n2sXhcrl4//33Wbp0qUxHjo2NZd68eYwcOfKMn+PZz+yvf/0rr7zyiswWjY6OJjU1tVtNP1tbWzGb\nzV0C+S6XS2ZWea7hVEQLJJFMMXjwYPr168e8efOkQNDpdNjt9nPyJLhcLlnLlJeXh81mo6GhQe5x\n0T0jMDBQeg4+++wzwK0QeSo9mZmZDBw4sMtlfD4WhLAcc3JyKCsrY+zYsVIIn+vvKS0tle8/JSWF\npKQknzRJVVVVli6kp6fLpAXx2ULR9QWhoaFkZGQA7u8sUsc9hZHn+BbxM9FVRXgtpk6disvlYuTI\nkV1c7b6oHdOSJDQ0NDQ0eiUXrQXlWXe0fft2YmNjZdPDkJAQampq+Oabb3jzzTcBt4aSkZFBSEiI\n1/29er1e1piEhIQQGRnJ+vXrWbt2LeBOQPDz8+PGG2/k97//PfDdJE9RTCjWIxIkhHbZ2dkptZTu\nIr63GMFtMBjk7y4qKpLTYr3ZG+2HYDQau4zZAHexdWFhIXq9Xmp9o0eP9nqxdUNDA48//rh8R+AO\n7k+fPl261gRWq5WTJ0/KTuG7du2iuLhYapkA8+bNY/Lkyd2KjQUEBDBp0iRWr14NuC21U0eheOJZ\ntB4bG8vo0aP5yU9+AsD48eNJTk4mICBAWuUmk+mc1+epWet0OpxOJ62trdLaFqn2nlYeIF20Yp/N\nmjWLhQsXkpGR0cU6PR/tWxQJv/rqq1RWVnLixIkuRaen4lmLKKyv8vJyvvrqK9atWyddqqNGjZK9\n/HyBqDMcMmQI5eXlKIoiR33cfPPNPvlM6NpMF75z9YqJ1a2trZSXl5OSkiLLTyorKwkODpbxcnBb\nmMOGDWPAgAHyvfqqCcJFKaBcLpf0ae/cuZPQ0FAuvfRSebFWV1ezYsUK9uzZI4OCKSkpTJo0iYiI\nCO+boX5+MuC7Z88e3n33XTZu3Nildcrq1avJzs4+YxbRqQLB8zLyVidhl8slYzmtra0EBQVRV1cn\n5wuJTLCUlJQez+IzGo24XC4aGhrks6ipqSEuLo7Ro0dLt58vLoz6+nra29u7PHOXy8Xf/vY32tra\nZPeN5uZmrFYrDofjtNlGQUFB3HHHHQA89NBDREVFdeudBQcH89BDD5GTkwO4Y2+nxpvETLSwsDDp\nJh42bBhJSUlceeWVpP//48vPVJh+PsJAURTmzp0LwN69e9m/fz8Oh+M0N6Gqql3cPpGRkYwdO5Zp\n06YB7q4RYtqwp8A4V5eoqqqyE0VVVRVtbW1s2LBBtjYbOnQoer2e5uZmqWQIRejjjz/m4MGDAJw4\ncUJ2VBk2bBjg7tLhi4C/QAjHjIwMtm/fjtVqlbFFX888E79br9cTEBCA2WyWdXO5ubkcPXoUvV4v\nMz4PHjyI1WpFr9czZ84cAK655hqCg4Ol68/z93qbi05AiYpxkWl2+PBhYmNjCQkJYe/evQDs37+f\nqqoqFEWRHZOXLFnC8OHDfSbphcYxfvx4vvnmG/z9/eUMnPfff5/o6Ogzfu7ZfiZ+LgKX3UGkGgth\nHRYWxoEDB/jTn/4kA99Op5PAwECWL1/u9SzHs+Gp+Tc1NXUZDjh06FAuv/xyQkNDfXpgRYsnq9Uq\nBU9JSQklJSVntVo8301SUhIvv/wyU6dOBfDK7CydTsewYcNk0sa//vUvOVxSFITb7XbGjBnDdddd\nx8SJEwG3Fms0Gs96cYjvcq7rE8WbY8eOBdzj5g8ePEh7e7ssjh8wYIAs2vUUPGIcjufIi7N953NF\nxNlEF+6KigrZJzEsLAw/Pz9p4Yn12O12WSALbmVg+PDhTJkyhYceekj+Xl/tOZfLJfdZQ0MDDoeD\ntrY2eTZFiyZfI+a/GY1G6aHw9/fn5MmTOBwOmfxSWlqKTqfjnnvukSNBxHDYnijovygFlGeig91u\np6qqioqKCll3EhYWxqJFi7jtttvkAfJ1ixxhOg8fPpxHH32UpqYmmWrcXRddd9Zts9nIzc3l9ddf\nl79n//79nDhxAovFIg9LUFAQ69atY8yYMT3eSSIpKYm4uDicTieZmZkAPPPMMz4XTuDOsExJSZFW\nFHS1YD3dWgaDgfT0dFmX9cwzz5Cent5lAJ431is+S1hGS5culSnkwhrQ6/WyJuvUS/9sazjftQnX\ns1DCBg8ezODBg7v1O72xroEDBwLQp08fWlpacDgc0kvg2UPOs+4xICCA+Ph4qTzOnz+fyZMnk5SU\nJN3LvrZgRCJHbGysFJaeyVE91UpIvFdxb2ZkZHDnnXdKZRXcGdHx8fFER0fLO64nW6BpSRIaGhoa\nGr0S3YWcN38KP3ghiqJIk7i1tVWmJwtNIDAw0Kedy78PzzqCC9Wi3nOs9J133sm6deuw2+1d0knF\nGoWvedu2baSkpFyQNYsJxHl5eTKVNT4+vkc0SUVRaGxsZMGCBbLnn0i3FUMBwV10es0115CZmSk1\nSV/1BDwbnme1O5/r6zhHTyGspLKyMn7/+99TUlLSpSO5SPoRfQBF8kifPn1kooIoSu2p5+HZdeTl\nl1/mj3/8I6qqyj53zz77bI9Nrz7b+jz/X9DN++z8/+LFKKA0vh9FUaSAuvvuu/n000+7FCgbDAbS\n0tJ4+OGHZXfsnuoa0ZvxrD2DnnVlaPz3IO7cqqoqnn32Wcxms5z83b9//x+F8nAKmoDSODNlZWU8\n8cQT7NixQxblPvXUU4wYMeKCzHfR0ND4r0MTUBoaGhoavZLzFlCaD0NDQ0NDo1eiCSgNDQ0NjV6J\nJqA0NDQ0NHolmoDS0NDQ0OiVaAJKQ0NDQ6NXogkoDQ0NDY1eiSagNDQ0NDR6JZqA8iEulwuXy4XF\nYqGjo4Njx46hKMppHQs0NDQ0NE5HayXgI+x2uxx9vWPHDlavXk3fvn155ZVXAHqkU7eGhobGxYwm\noHyA0+lk586dLFmyBHDPF7JYLJSUlLBx40YAFixY0GNt9S9GxNTTlpYWqqurAfcgSqPRyPDhw+Xs\noZ5s9Knxw/DsNffaa6/x0EMPydlNGhrngubi09DQ0NDolVy0FpTneG6z2UxNTQ3bt28HoKWlhczM\nTKKiomQX79LSUg4ePEhFRQVlZWUAzJgxg8cff9yrE2QVReHDDz/k6aefltNPxcjkyMhIysvLAbeV\npVlQpyPicxaLha+//poNGzbIqb8tLS0kJydzww03cO211wKcNj78x4qqql3il06nEz8/PwwGg+y6\nrihKlwGLgp5+Ni6XC3APWnzvvfdoaWnh5ZdfBtD2fC9HVVVcLpccCutyuTCZTPj7+8vm0j25ny5a\nASXw8/PDbrfjdDqJjY0F3AKhvr6ekJAQmpqaANi8eTPffPMNbW1tOJ1OwD0Z1OVyeWU+jhCYtbW1\n/OUvf6G8vFxeJlFRUUydOpXLL7+c6dOnd/nzGt/hcrkoLS0F4LnnnmPLli00NzfLuT8A9fX11NbW\nyjlWU6ZMISgoyKuHRlEU7HY7iqLI2Tzfd7GKuUPgmxEdLpeL2tpa3nzzTRwOBwARERHExMRgsVik\ngj6WudwAACAASURBVJWTk0NkZCQmk4lrrrkGcI9iNxqNXl/T2VBVlZKSEgDWrFlDa2sr69ev54UX\nXgDwqjL4Q9cjBKbNZsNisaCqqnyfJpPJ6/tHzITznJR7JiVKp9Oddg+4XC7sdjs2m42AgADAPQpH\np9P5ZG95Ph+Hw4HFYqGoqIh169YBkJCQwIgRIxgwYACRkZEAXZQiX3PRCihPTTE6OpqoqCg5Atpz\nNHZlZSXgHoAXGBhIa2urFFA1NTW0tbV1eyQ7fKc1fvLJJ1RVVaHX6+W4+SeeeIJLL71UDlIE90s+\nday4N/Hc+GJAoefPXC4XiqLIC0+MxjabzXLwY3BwcI9tRIfDwY4dO3jqqacAOHLkCCaTiZSUFKl4\niLXV19ezYsUKucasrCz8/f27vVbxfBwOB0ePHmXv3r2MGzcOgJEjR542BFMMzszPz2fs2LEAcpy3\nNxB76tixY6xcuZL8/HwyMjIASExMJDY2lvz8fDl8r3///nz00UcUFxfz1VdfAbB8+XJSU1N7ROtV\nVZXm5mYef/xxABobG1FVlYCAAHnmfPW5LpcLp9OJxWIBoKGhgaamJnbt2sWWLVsAyM3NxeVy4e/v\nL9cTExPDI488wrx587o9KFAopDabjebmZgoKCuT9M3DgQFJSUggMDJR3E7jjqo2NjRw8eBCAXbt2\ncfjwYfz8/IiLiwMgKyuL22+/nQEDBsg9fj4DBD3Pv4jxikxjcCv2QiEUd6KiKHz55ZccO3aMBQsW\nAEhB1RNctALKE/GyPC8oo9GIoigkJCQAMG7cOAoKCmhtbZUPf8SIEURERHT78CqKQm5uLgBvvPEG\nzc3NhIWFSS32yiuvxGQydfkcPz8/abl1R1B5Ch7Pvy9+Zrfbqaio4Ntvv5UHo7W1lYaGBmw2m3RD\ndnR04O/vj81mk4L1scceIyoqyueXm9Vq5dlnn+Wtt96SF0zfvn1ZsmQJ2dnZUpOsrq5m3bp1/POf\n/2T37t0AhIeHM2zYMK9aCR0dHbz11lscOHBACvChQ4dKpUI82+bmZm655RZsNhvvv/8+4D0Bpaoq\njY2NgFvJMpvN/L//9/8YP348gHwml156qXw/drsdu93Oc889J/fjli1b+PnPf94jAsrlcvG3v/2N\nw4cPA8gLf86cOT6x4oRAsNvtHDx4kDVr1sipyLW1tbS2tmK32+WUXTGU0+l0yrNQX1/PqlWrGDdu\nnJyefD6KjqclUlpayuuvv87WrVuJiIgA3K7olJQULBYLR44cAdwu67q6Omw222nTkv38/GQ4oL6+\nHlVVeeKJJ+TddT6uUlVV5X6uq6ujs7OTiIgI+VxUVZVu45MnTwLuZ9Xa2opOp5NnMyIiosemMmtJ\nEhoaGhoavZIfhQV1NhRFoaWlBYCCggKSk5NJTk5mypQpADzyyCNSE+0OTqeTP//5z/JzFEUhKSmJ\nX/3qV4DbDXWqv9nz34UmeL7B/o6ODvk5wjITcZs9e/bw3nvvceTIERmP0+v1BAcHExMTI2M5KSkp\n6HQ6mpqapBYqtC1foKoqDQ0NANx2221s2bIFVVWZOXMmAH//+9+l60o8k6CgICIjI7HZbHKkfXt7\nO3q93quuyKKiIj777DOioqIYNmwY0FWrFu9r37597N27l9TUVK+4iU9FJPiEhoZy7bXXMmHChO+d\nguzv78+IESO6rKW1tbVH4p2qqtLS0kJ9fb2Ms+bk5FBVVcWAAQO8Pr1ZURSam5sB2LlzJ4888gjN\nzc3Sgg0KCiI2NpaUlBT5PNLS0mhqaiI0NFTuve3bt2Oz2WhsbCQ1NfW81+MZVujs7GTv3r3U1tZS\nUVEBuF36x44do6WlRZ5Nm82Goij4+fnJNSYlJREbG4vNZiMoKAiA7Oxs7rzzTpkQ9EPXI86NSLCx\nWq3yrjhw4ACVlZVkZ2dLl51Op6O9vZ2jR4+ya9cuuZ729naysrLkerQkCS+gKAodHR18+OGHABw6\ndAi73c6wYcP43e9+B+AV9x64BYR4oU6nE4PBwNy5c7tcsN/nM/YMUiqKQmBg4A9el06nO82tpNPp\npNkeGhrKhAkTaG9vJz09HXDH4/r06cP48ePp06cP4N7EbW1tvPbaa9Ilci7rOBdUVeXkyZMsWrQI\ngKNHjxIQEMD999/PAw88ALhdIkKIiwvW6XSya9cuWlpa5EHt379/t2MHAvEe3njjDZxOJ6NHj5ax\nJXHBel5Ey5Yto6Ojg4yMDK+tQaCqqqwdSkhIIC4u7j9e8oqi8MEHH6CqKomJiQCkpqb2SMao0+lk\n9+7dDBkyhPj4eADy8vKIjY3FarV6fR85HA7q6+uB/4+98wyPskr7+G9aGukJKQRIAoSONGlBuiAo\nSpOlWVBWFlbEBRaFfaUsIgrqpeKuXSwIAqIgKKCC9CoE6RBaQkISSK/Tn3k/PNc5zkRAJDOh7Pw/\nQQjznDnPOXf93/cNb7zxBjk5OYSHh9OqVSsAhg4dStOmTQkNDSU4OFj+P8GoFTmfgoICLBYLvr6+\n8pzdSPhKo9HIPa5ZsyaDBg1i/fr1kjDiTMoQhofBYMDf35977rmHl156CVCVqKIoZGdnSwOxXr16\n+Pv7/2kjzLljjcVioaioSDKdd+/eTWlpKb/++quUCwaDgVOnTkkiEqjvtUuXLnTu3FkSXKqTNXvH\nKqiysjLWrVsnE6QGg4GQkBBeeOEFIiIiAPdYAoqicOHCBS5fvgyoQs7f358BAwZck5bpnNgVVs03\n33xDRUUFo0aN+p33cC1c6XfEhWjdujXNmzdn8ODBco3+/v4EBQW5UEdBPcSpqak0btwYwO1CF9Tv\nfeHCBZ555hnOnDkDqJdy2bJlNG3aVK7b+TuJC/2f//yH9evXY7VaJXHioYcewmAwuIWFKd7DqVOn\n8PX1ZdCgQdLDFp6pxWKROZadO3diMBho37692xWARqORz46Ojr4uTz8nJ0caSuLdFRUVSSXvSaFS\nUVGBwWCgYcOG0gMvLi7GbDaTmZkplb+79kmn08l73KxZM8LCwoiJiWHKlCkAxMXFybyXs+Lx9/fH\narXK/F5eXh5JSUnExMRUyQt3fl9169ZlwoQJdOzYkTVr1gC/3S1FUaTx2KxZM+677z46d+4sfwbI\nCIy4m76+vje0Nuf3rdfr8fX1lffGZrNx7NgxcnJyJKXc19dXenVizyIjI+nSpQuNGzd2WWN14Y5U\nUDabjZUrV/L666/LxF7Pnj2ZNGkS9erVc+tFtVqtTJ48WVrVGo2G++67j2bNml1VMcFvlNe8vDy+\n/vprAFauXElaWhpr1qzhzTffBKB58+ZVOpw6nQ6dToePj4+0JCsqKtDpdC4HUbRmCg0NpV+/fgAe\nOZBGo5FXXnmFAwcOyIT/Rx99JEOMlYkeJpOJ559/HoClS5dSXl6On58fw4YNA6Bp06ZuCe85HA6O\nHTsGQEZGBi1atCArK0uGEn18fKioqODcuXMsXLhQ/p+aNWvSs2dPj7Id/+i8Ckt78uTJ5Obmotfr\npRfjKXqygDg/5eXlhIeHyzAyqN5JRUUFcXFxVfJOrgSdTic9zCeffBI/Pz/Cw8Nlh5HKjEvxbEVR\nyM3NlSzHixcvMmjQIEJCQq5YQ/Zn4Pz//Pz8aNOmDXFxcQDs27eP4uJiQkJCqFevHgCDBg1yUURi\njaK2UxAsbvT9VVZQwcHBtG/fHlAV+HfffceSJUtkuLOkpER6XcLAiY+Pp0uXLi7hxSuRsa70THfA\nS5LwwgsvvPDilsQd5UEJ7b9r1y7mz59PZmYmzZo1A2D+/PkeoUxnZ2dz5MgR+bn+/v40btzYJZTh\nbGWI+osDBw6wb98+LBaLtNwF9Xv//v3MmDEDgEWLFslQRlXgHDcOCAjAarXicDhk2K+wsJBTp05R\nq1YtunXrJv+PuyBCPBcvXuTUqVMAJCcnAxAcHIzNZnPxoBRF4dKlSzz11FPs27cPUPcuMDCQYcOG\nMX36dIDf0fdvFA6Hg23btsm/Hz9+HJ1Ox/79+wHV67x48SL169eXvQGDg4MZOHAgMTExVSa6VIbz\nXtjtdqxW6xU/W1EU5syZA8C6devw9fWlYcOGsn6rX79+N1Qzc70QZ/vixYtERkZiMBj47LPPANWD\n0uv1JCUlud2L02g0MoTXsmVLmfu91nM0Gg02m42NGzfK0FtISAj9+/d3S5i48rNEETCo5+ny5ctk\nZ2dL4pbBYKBr166yFAbU8yyK1Z3JCzfyfAHhtfr4+Mg9S0pK4plnnmHkyJHMnTsXgCVLllBaWgr8\nVlA9cuRIoqKi/vBceyqEfMcoKEVRSE1NBeBvf/sbWVlZhISESCZdeHi4R0Id27dvd4kr169fn3r1\n6slwAvwmbOx2uxTOCxcupLS0lNq1a8vYdefOnSksLGTPnj3s2rULUEMDIuRWVYgDJC6y3W6XYZKs\nrCwyMjIIDg72CCPNuQjWZrPh7+8v9+Lll1/GZrMRHR0tmVlZWVls2rTJhRARERFBu3bteP7552W4\n0l3v1Gq1ypBKfHw8Go0Gi8UiFZTZbCYuLo6AgACZE7Pb7YSFhVFeXi7zV55ovWQ2m9m7dy+tW7d2\nCfsoisLBgwdZsGABoN6BPn36MG/ePEl+EQxST0GEF/Py8jAYDHz11VdS+BuNRmJiYmjXrp1H2uQ4\nh+Sup87K4XBQUFDAwoULZej/sccekwWx7obNZpN51g0bNpCWliYNNYAffvgBRVFk2BqgYcOGaDQa\nhg4dSsOGDd2yjiu1vRJ7Fh0dLc+PwWDggw8+AKBFixaA2okErq6APN1S645QUOIg/P3vfwdUr0av\n1zNw4EBGjRoFeKYFjcPhIDU1Fb1eL5OP8fHx6PV6rFarS9cI8fvCOrJYLAQFBRESEsKAAQMA1Xpa\nv349e/fulQJPsIDcDSFExcXIzc1lz549zJgxwyOML6Gsy8rKuOuuu/D19ZVFwgcOHKCgoAAfHx8p\n8IxGI3q9nqCgIHlRtVotdevWdRv7sjI6deoEqFZ1WFgYsbGxLiSJoqIi9u3bJzvSJyQkoNfrKSsr\nk+/4RthWV4Mz82vNmjW8//771KpVC1DJIZcuXWLixInSKw8LC+OLL75we+uea8G5xdO3337Ll19+\nKVmgGo2GYcOGER0d7fH1XM/nK4rCggULSE1Nlcy1xx57zO3ek3iWc1GuMNAMBoM8UxUVFTLvI55/\n8uRJzGYzRUVFPProo4D7ogSVUTnn1rRpU8LDwzGbzfKcCWP2ZvW6vO0VlKiO/vbbbyW7ymKxEB0d\nTXJyskeZJ4Ka7VyZnpGRQUpKCl27dpXPFglbnU4niQGpqamkpaXRvn17WrduLT9z9+7dUjmBKrw9\nCSFMy8rKyM/Pl+wmTz2ndu3a9O3bl3vvvVcK9aNHj7J3714yMjJk+Cw2Nlaym06fPg2oiiMuLs4j\n9VkGg0G2ymrcuDFardZF0dhsNnx8fDh58qRMKsfHx9OjRw/q1q0rQzlXSs7fKMTn1KhRg7FjxzJu\n3Dg5Y2zr1q0UFBRgNBrls+fPn1+tygl+M74iIyM5f/68NDrEumfNmlVt67lavZf4+aFDh1i8eDF2\nu52mTZsCVKn26WoQLcSc+4N269aN4OBgGjVqJIkKJSUl/Prrr5hMJrlvX331FadPnyY7O1tGUR58\n8EGPlXs4l3HUr1+fhg0bcvjwYSkHMjIyZMnCtSBKQrwkCS+88MILL/4ncNt7UKBa/1u3bpWhBa1W\nS1xcHB06dPDoczUaDUOGDGHfvn0yFJeWlsaqVasIDg5m3LhxAERFRUnrQsTKz549S0pKCqNGjZJW\nqNlsZtWqVVitVmkVi6p8T0GEkc6ePUtAQACXL1/2SBNb4Y0EBgbStWtXDAaDDE116NCBdu3acfr0\nafl77du3p06dOhw7dkx26QgICCAyMrLK/Quvtj6RGL5arD0gIIATJ07INcbFxdG6dWuXEIwnLF2t\nVkvTpk35/PPPZfnBiRMnuHTpEsHBwUyYMAGARx999KaFYqxWKxcuXMBiscjzPHToULc2z/0jCHo2\nqF65zWbDYDDIPOK7775LcXExer2ezp07A7+df3da/6K+0Waz0bNnTwDuv/9+fHx80Ol0Ls9p2bKl\nrK8DtYdgeno6Go1G5q/c2QmkcsNYRVHkHjRt2pQuXbpw8eJF6dFlZWW5pCv+6LPdff5uewWlKAqH\nDh0iJSVFxnZr1KjBkCFDCA4O9mjHcI1GQ/v27ZkxYwYzZ84EVEGfl5fH66+/LucYvf766zRo0AC7\n3c7atWsBNUEaGhrK4cOHJVFh6dKl7Nq1C61WS5s2bQDo06ePxxgyzoK+bt26mM1mWT/jbghlpNVq\nMRgMLnFtQTZo37499evXB9RcjiiKFZc3OzsbnU7n1jCawB8le7VaLaWlpRQXF8t82tChQ2+4iPLP\nQqvVkpCQwIgRIwBV2B4/fpw6deowZswYwDN1a38EcX7279/P2bNn0Wq1soamuhWmRqORoakNGzZQ\nUlJC48aNZc1TTk4OWq2WiIgI7r33XgCZf/LEOkNCQn4X5r/SmjUajVQAolmrn5+fbMnmLpaqc0Nb\n8TOdTiffV3R0NOPGjePo0aOyWeyOHTvo3bv3dZ1zT9yD21ZBOc9fmj9/PjabTeYQZs+eTZcuXTyS\n/KwMPz8/evfuLZOKM2bMkN6cSKZ37dqVXr16ERYWxo8//giowrawsJCFCxfK0REpKSkoikJERIRs\nbe9cQOgJiH1MTU3F19eXnj17eiRhLARHaWkpISEhBAUFufQijImJcbnQFouFn3/+mZUrV0rGVUJC\nAu3atcPPz8+jSeMrweFwcOjQIUmnBpV1WV3jSMC1rVVeXh7BwcE8+uijHmOhXe+aQO35mJ+fj0aj\nkXehfv36Lha6p6HT6ST5YebMmVitVsxms2SL7t+/H4fDwYQJE+TYEk+sTSjpsrKy62YXCvbqiRMn\nMJvNREVFyYJed551YSgqinJFAkRISAjdunWTLeJSUlLYv38/ycnJ0gGo1hxntT3JjXA4HFJovfHG\nG3L0Qu/evQG455573Mqm+iMYDAZatmwJwOLFi5kzZw5ffvml7BVWUFDAN99848JSs1qtWCwWWXcA\nKgurXbt2JCcnM3z4cMAz7YacIbyB9PR0atWq5RGGnNVqleSHAwcOEBERQevWrV1aufj5+WG328nK\nygJUD/OTTz4hKytLNrR94oknqFu3brUO4BNQFIWtW7dis9lk6NgdjYb/DGw2G9988w2g7mnLli15\n9NFHq1VJVoY4P5mZmVitVgwGg+xfGBwcXO1rcy6l0Ov1+Pj40LdvX0BtJVanTh1Gjx593WfoRiIw\nQuiLkT9/BKvVyrx58wDYs2cPdrudBx54QEZWPEXNr6ycRa3U4MGDZcnOnj17eOONN7DZbLKRs7sb\n/14LXpKEF1544YUXtyRuSw/KaDTy7rvvAupYBqPRSHBwMKNHjwbUZPrNstxCQ0N5/fXXmTt3rhxz\n/dNPP8kBYcJjcm7gKSrG7733XmbOnCnra5w/11MQ6zl06BD16tWThaDugqIo2O122S1j7969FBYW\n8vPPP0t6fUBAACEhIRw8eFB2Wz5y5Ag2m42mTZvK/F7Hjh1vSp4FkF03atSowQMPPAB4Jjx0redf\nunRJUtxLS0sZOnSoJNPcLIiQkWiEqtVqZTThZr0rZ1itVukNiPBjcHDwdd+rqnRx8PHxkZEeh8Mh\npz6LfxfkiFdffVUWyFosFmrWrMnkyZPd6qmIZ4qIjN1uv2KvQiGPxPDG77//nvT0dJo2bUrHjh0B\n/tT+VRW3nYIymUy88MILMm9jNBrx9fVlxowZVZqI6U4IxpcYXz5r1ixZryUEzPfff09GRga1a9eW\nbWni4uKIioqq1rqR48ePy7/fddddHpt8Kjom5+fns3fvXoKCgti5cyegXuRLly5hMplkElfU/kyY\nMEF2jajO0EJlWCwWAgICaN68uSwcrs5YvMPhYMOGDWzcuBFQlXr79u1vGmtPQDzfYrGg0WiIi4uT\nzNPKjLWbAavVKju8m0wm/P39rzqF2p3QaNQmvcKIyc3NlU2axTkuLi5mxowZ/Pzzzy6zvz744ANq\n1arlESKQ+N46nc5lH+C3RrWlpaVyVH1RURE2m41ffvlFrrE6cVspKJvNxtq1a/n0009ll2lfX1/G\njh3LxIkTq9Wi/bMQ8V2RQP7rX/96xd+pTiiKwqeffiqfPXLkSLcrdzGM7aGHHgLU5H5BQYFM8gsk\nJSVRq1YtmjRpAsBf/vIX4uLi8PHxuelCDtQSgNLSUnJzc13WXV3QaDS0bNlSdseuUaMGfn5+vxMy\n4ned4clRG+K83HvvvWzZsoW2bdvKNd5MgwJ+K6QX4z8qKiqqNW8oJgmItezYsYPdu3fLbjInT56k\ntLQUh8MhiS7Tpk2jX79+HpNlf3QOFEXh/Pnzco1arRY/Pz8aN24syzCqE7eFghKJRhEu0mq1Ukj8\n7W9/Y/bs2be0croSbgWhW1BQwNmzZwHVIo+NjfUYO040xJw+fTrPPPMMJSUlLqMZEhMT8ff3d5kH\ndSvskRD+er2e2NhYioqKCAwMvCnrOHbsmPQwy8vLSUtLIzExUSoJh8Mhe/Q5J8OdLWfxM3dBeNxL\nlizBbDZjMBhuumJyhjOt2mAwuAwm9DQ0Go0MqYWGhtK6dWvS09PlsEQRTmvSpAn/93//B6jttjyx\nf1cyUpxDjc5/Dw4OlsNEGzdujKIoPProo9esE7zaM6oKL0nCCy+88MKLWxK3jqlzDQgL0cfHh+HD\nh0sKthc3DtEvrGvXroDa7d2TCXfn0GFISAghISGy4/btAI1Gw2OPPcaBAwekVezpKbWVn9+kSRPZ\nO65evXp07Njxitb2lbxPT6/TuRPHrQIRVhe0d19fX+rUqVNtOWrn8xEQEED9+vUZPXo0d911F4Ds\nzVezZk3piXrK+7zW+3f+Nz8/P5o1ayZD7cOGDbuuUSZ/9Iwbhaa63N3rwC2zkP8F2O12jhw5ws8/\n/wzAgAEDblqN0e0A0b6mcvH3zQpDVqdyvJ0h2I+gEqrCw8MJDAy8KSkB0VpIhB1FLdKtEs6+EtxE\nJrnh/+xVUP+jUBQFs9ksvYGbzXz0wgsvbk2I8oEqwKugvPDCCy+8uCVxwwrKazZ74YUXXnhxS8Kr\noLzwwgsvvLgl4VVQXnjhhRde3JLwKigvvPDCCy9uSdwWdVBeeOHF7Q3n2V+CVn2rUqu9uHXgVVBe\nePEn4K0/+nOw2WxcvHiR7777DlAbtnbu3JmGDRvKLv43cz893TTWi6rBG+LzwgsvvPDiloTXg/Ig\nRJNbs9nM4cOHWbRokaxqb9myJRMmTCAiIsJbJHsbQFEUOW5AFDdfq2mms6f1v2adC6+koqKCmTNn\nsmzZMjkXKSgoiPT0dMaPH0+NGjUAZHeO6hwzI6YhbNq0ifz8fLp37079+vWB/+2idfHuKnuWN+sM\n35EKqry8nO+++469e/fKXnP3339/tQ5QUxSFsrIyAD777DOWLl1Keno6ZrMZUEcpnz59mrfffpuI\niIhqW9e1IBSq8yEVY+mFYtXpdNhsNrZv387f//53wH0XuvJsGvF38fk345KItjQmkwmTyYSfn5/c\np2utS1GU267Dvrsgzvj999/Pvn37sNvtcvpAQkICoaGhGAwGrFYrQLW211IUhbS0NKZMmQLA7t27\nCQwMJDAwUCood8H5PCuKcsVzcyvm4kSfTvhtsKGfn99NUdx3lIISVtqkSZNYs2YNFotF9prTarX0\n79+/WjZZURRycnJ46aWXAPjuu+8wmUyEh4fTqFEjQB1glpqayieffMLkyZPlGt2FP5oRdKXfF4MF\nhTI6c+YMO3fupKCggOzsbAAuXbpEVlYWCQkJUkFVdY02m43c3FyWLFnC4cOHAUhPT6dWrVp06dKF\nQYMGARAREVGt86EURSE9PR2AnTt3EhMTQ+PGjeXsHmHwVBZE4juZTCZAnd10qwkhT8FqtTJmzBgA\ndu3ahaIoREVF8dprrwGQnJyMwWAgNDRUNpetrrye0Whk5cqVzJw5k9zcXEAdtFheXs4vv/zCsGHD\n3PIcMfjPZrPJuzR79mw2bNhAeXm59MB9fX3R6XQ0adKEbt26ATBo0CDq169/U3piKooi1/zFF18A\n8O2331K7dm1eeOEF2eS2OhXVHaOgHA4HR44cAeDnn3+mtLTURcDs2rWL3r17e7zjssPhwGQy8cYb\nb7B69Wr5s44dOzJy5Eg5DjszM5OvvvqKrVu38vjjjwNQs2ZNj6zpei6/oihkZ2ezfPlyTpw4IX/u\n7++PXq8nJCQEUC2q3NxcYmJiqiRUHA6H9DBfffVVtm7dSnFxMbGxsQAEBgZitVo5dOiQ9GLat29P\nmzZtqsUTttvt7Nmzh7lz5wIQHR3N2LFjiYqKks+vPLob4Pz58/j7+7uM9tZqtW7pFO88mbi8vFzO\nXhJnunIj2+qG3W5nyZIlfPXVV/LvwcHBrF27Vp57rVaL0Wh0WafzXrkb4lwDPPHEE+zfv192ORdr\nDAgIwMfHx21zojQajctMLoDevXuTnZ3N0aNH5TssKSnBbreTlZXF1q1bAZg/fz6dOnVi0aJFcoaa\nJ9+pUKager6XL1/mlVdeYcuWLYA6YDQtLY2nnnqK5cuXA+rstuo6Z/+7wVYvvPDCCy9uadwxHpTd\nbmfZsmWAmpyNiYmhoqJChv3OnTtHSUkJPj4+Hs0N2O129u/fz5YtW2Qsvnfv3sycOZPExEQ57yUh\nIQGbzca7777Lpk2bAHXMubvcZ+dJoleyUMWUVWH5Hzx4kLlz51JSUiLHYnfo0IExY8YQFhaGzWYD\noKioiKKioipbUGazWYZA161bx3333cezzz5LaGgo8Ft4bdWqVRQWFgLIsIynYbfb+emnn3j99dfl\n937iiSfo0KHDFc+Ow+Hg3LlzAPzrX/8iISGBqVOnyrzLjXp8wqIXXuubb77J119/DUBOTg5WUG+R\nyQAAIABJREFUq1VOmAZ1lk+dOnXo2rUrDz/8MKDOjapRowb+/v7yver1erdbwA6Hg/T0dCZNmiT3\nLCQkhG+//ZbWrVvLNYrO2M6Wu6dCRlarlZSUFJlvOn78OLGxsdx///1yb7/44gtsNpv0atwF8c7j\n4+MBqFu3LsOGDcNqtUqPbvny5ezdu5fMzEwuX74MqFOuU1JSePHFF3nllVcAlVjiCY/FbrdTXl4u\n79e3337L/v37MRqNtG7dGoDi4mJOnz5NZmYmjz32GKDuWUJCgtvXcyXcMQrKaDSSk5MDqIfhnnvu\nITMzk4MHDwJqcr+4uJjw8HCPKiibzcaWLVuoWbMm9913H4AUvM7DyBRFwWQyERwcTGRkJPD7vFFV\nIC693W6XhANnQaAoCgUFBXK0s7i8zzzzDA899BCgChgRNhJrc65dudFL43A42Lx5M+vWrQOgTZs2\nzJo1S7K6xPrq1q2Lr6+vVPRJSUkeHScuBOYvv/zCmDFjqFGjBrNnzwbgnnvuueq5sdls/Pe//wUg\nNTWVhg0bUrNmTbnWG9knh8Mhhea7777Lm2++SV5enhT+ziQSsW6r1cqJEyc4efIkn3/+OaDm7erW\nrUvTpk15+umnAWjYsKHbQ91Go5EJEyZQUlIi8ycvvvgiycnJLufOarVSUVGBoijSGPGE8LXZbCxd\nupQ33niDgoICAAYPHszUqVOJjo6WZ2/p0qUoikJCQoJHw1Zi9pNOpyMxMRGAf/7zn+Tl5WEymWSu\nasGCBezdu5ejR4/K0fDJycluk1nOZ2bPnj3s2LGDzMxMQJUZ3bp1IzExUSpWPz8/cnNz+de//sXu\n3bsBGDt2LN9++221DKi8YxRURUUFpaWlgErh7tSpE999953Mc5w6dYqcnBzq1avnkec7V8qXlZXR\no0cPHnzwQUC1gITVKJLoxcXFbNu2jejoaMkecucFcVZ2gpEjPl/E5ceMGSNJCe3atWPp0qWEhoZe\n0aJ1pptWNaltsVj4+uuvpZW5YMGC3xEJtFoter0eg8EgDY+wsDCPCBGHw4HZbJYX8IknniA0NJRJ\nkyZJBX41AaEoCj/++CM7duwA1FzViBEj5CC6G12P3W6X61myZAnl5eUoiiIT7P7+/i7vU8Bms6HT\n6aTSKigooLCwkIyMDIKCggD4xz/+Qe3atd2yl4II8s4777Bp0yY0Go08z8OGDZPnXpzBjIwMioqK\nqpzDvBqEAv/www+ZPXs2drudfv36ATBr1izCw8M5f/48M2fOBKCwsJCwsDCSk5OrPX+n1+uJjIzE\nbDZL2eXv7094eDg1a9aUHq/NZnNbnk6ci9OnT/Paa69RUlJChw4dABg3bhxRUVH4+vq6PCsmJoYp\nU6ZIEklKSgq7d++mR48eHt+zO0JBORwOUlJSZAioQYMGpKWlsW/fPoqLiwE1gRweHu6xcIJzQjwq\nKoq4uDhpQZtMJgoLCzl+/DjHjh0DVEv7xIkTjBw5UhID3PmyheDQaDTk5+dTXl4u98LhcLB8+XLK\nysp48sknAZgzZ851W0RVpVDn5+ezd+9e6TkGBgZe8fc0Gg2FhYXyu3gq1OFwOMjIyOD1118HIC4u\njiVLllC3bt1rnheHw0FOTg5z586VodJZs2bRvHnzKp8zq9UqFXNERAS5ubmEh4dLL6hXr17Url1b\nCjFQz3h+fj5nzpyRSfcVK1aQnp6OwWCQ+5idnU3t2rWrtD74bd8A3nzzTex2O35+fgwePBhQ35ei\nKBiNRn788UcA9u/fT2hoKIMHD3b7XVQUhe+//x6Al156idLSUpo1a8b48eMBlTW3fv16Zs2aRVpa\nmvwOBoOhWktQBESIPTU1lUWLFgGwfft2NBoNCQkJsvxEGCxVjR7Y7XaOHj0KqEbhqVOn6N+/P889\n9xygRkyu9E60Wi2tWrWS97SkpIS1a9fSrVs3j5dSeEkSXnjhhRde3JK4Izwoq9XKe++9J2P2Fy5c\nICsri4sXL8rQwoABA6hfv77HXVKdTkeXLl04cOAAO3fuBNT8zpYtWzh16pRLOCYqKor8/PzfdShw\nB4S1lZWVxZ49e9i+fbtLIjg0NJSXX36Zzp07A/ypeHJV6eVnzpzBbDYTFxfnstbKUBSF8+fPy4St\nO/fHGRaLhbfffpu8vDwA3nvvPeLj46/ZKQJUL3XixImcPXtWhj86derkljzZ5cuX+emnnwB1Hzp3\n7szQoUPp06cPoL4vnU73O4s3JiaG0NBQ6SEYDAbq1KlDp06dGDJkCOA+mrDdbufNN98E1FCZn58f\nffv25amnnpL/npWVxeLFiyW5o7y8nLZt2zJs2DC3h7RLSkqYN28eoIbQo6OjmT59usyn7Nq1i3/+\n859kZGTIe6jT6SgtLeXcuXPynFVXqK+kpIT//Oc/rFy5krNnzwKqtzJw4EBGjBghaeZ6vb7Knoqi\nKBw4cICRI0cC6n498MADLrnfq31vjUaD1WqV3Tc0Go3cU0/3MrwjFJTdbufcuXMyvlpYWMi5c+co\nLy+XIZDnnnsOPz8/jx8+rVZLYGAgx48fl3mJtLQ0KioqZDhBoKysjBUrVtCiRQtAZfu5Q7g5J9Bz\ncnI4ffo0x44d48KFC4Ca+OzUqROBgYEutTzXszdVzT85HA6KiooICQkhKSnpmr9bUlJCWVkZrVq1\nAjzD9nI4HOTm5vL999+TnJwMQLNmza6pnMQ5e+GFF9i6dSsJCQky9OaO92cymThw4IBM7sfGxvLQ\nQw/RsWPH3zHxKq9TvHch8AICAhgyZAj33XcfjRs3Btyn6EtLS9mwYQOgstaGDx/OrFmz5L8fOnSI\nDz74gG3btkk2bWhoKAEBAYSHh7v1LiqKwurVq2VY1NfXlz59+tCgQQOp6J9//nny8/PlegHCw8Px\n9/d3Yb16GmIvJk+ezFdffYXJZJLrmT59Oo899hg1a9aUsuJGSTbOKC4uZtq0aTLX1bFjR6ZOnUpA\nQMB1FfHv27fPpfBcGLaexh2hoEC1FCsqKgA4efKkZAqJWHtUVFS1WEYajQZ/f3+OHDki6aSKolCz\nZk169+5N06ZNAdWSPHr0KEePHpUdGV577TUGDx7slnUKYR4UFERMTAwJCQlSeMbExFBYWMgnn3wi\nE6RDhw51UVgCzsrO+TveKMT+dO/enb59+wKubWCcySYbN24kMzNTsp48AYfDwenTpzEajbLLx7Ws\nVbvdzoEDBwD48ssvCQgIYOLEiZJ8U7kP35+F3W6nsLCQs2fPSss2MTGR1q1bExkZ+TtmYOV3Y7PZ\npMcOqiDu0qULjRs3lsrNHXkDq9XKypUr5d+HDh3K/Pnz0ev1smB+3rx57N+/H6vVKgkaUVFRJCYm\nVolEcrX1nDhxQn7HqKgounbtyrZt21iwYAGgGjw6nY7o6GgpYC0WCxqNhtatW1eLfLDZbDJPtn37\ndmw2GwaDQTJ+J06ceF1K44/g/P9FR/lTp07J/RkwYAARERHX9RybzcZ7770nlXidOnU8eiedcUco\nKL1eT2BgoAzRaLVaWW/Rtm1b+TvVAZvNxs6dO2VHAUBalqGhoS4HQgi7ESNGAGoNTdeuXavcUUJQ\nWkGlZjdo0IC//OUvUoFfvHiRRYsWsXPnTilsmzVrRtu2bSXrSnwX57oVcE+3gmbNmrl0E8jLy5MK\nVaw7Ly+PxYsXU1hYKFsLeUqAFBUVERgYKK1v59oiZyiKQmFhIR9//DGgWundu3enXbt28nesVmuV\nPBRFUTh27BhZWVnSmOnduzcJCQlX3XvxbiwWCxkZGcycOVN2UGnbti3h4eEuYaKq7qPwOo8fPy4Z\nclOnTsVgMHD69GnpRf3666/4+PjQunVrya67cOGCVBTugmAJlpeXS+p6o0aN2LdvH6tWrZKeaEhI\nCN26daNbt27UqlULgNWrV5OUlESdOnWqRUGJ8glQ6f7FxcX4+/szYMAAwJWd6U68+uqrmEwm2a6o\nW7du6PX63xk4wrhy7seZmprK3r175e/07NlTGrMej0h59NO98MILL7zw4gZxR3hQOp2OHj16sHnz\nZuC33IVGo5GU1+pKfDo3qBVNTufPn39FGqter6d9+/b83//9HwAzZ87krbfeYs6cOVXOt4j/L6xu\nX19f2U8vOjqa6OhoRo8eLS3J+Ph4rFYrly9fljFyu91ObGyspAuLz61qiC88PJwmTZrINebn53Py\n5Elq1Kgh13jgwAHOnj17VW/GXdBoNDRo0IDg4GDpTV6+fJnY2NjfFTYLSvDJkycBaNy4Mc2bN3ep\nxq/sJf9ZGI1GlixZQmBgoCSRtGzZEoPB8Lt9qDwa4eTJk0yZMoWUlBQZMYiIiCAoKMgtzUedx2h8\n8sknXLx4kQkTJgDq987MzGTKlCn88ssvgFo+0LVrV4KDg2X+Mzw8nOjoaLdGNARBIicnR+59SEgI\n6enpWK1WWc4wZMgQBg4cyMGDB6UXXLt2bR5//HEXur4nYTAYpBfTunVrDh48iMVikb0KPdUx4vz5\n82i1Wvk9RRixsgclegiKn5eXlzNr1iyXfXz66afx8fGplqaxd4SC0mq1jBgxQoYR9u3bR0ZGBgaD\nQb746oLJZGLPnj3YbDamT58OXLvVjUajkQywl156id27d2Oz2dxWl3G1uobQ0FDatGnD/fffD6jE\nCbPZTHFxMUVFRYCqQOPj42UFvFhvVeHr6+sSw1YUhUaNGskO0KAKndLSUgwGgwvz0d3QaDQkJSWR\nnJzMDz/8AKghtUGDBnHffffJWhSr1crhw4f59NNPSU1NBVSlfuHCBdLT02nQoAFAlQWdj48PWVlZ\nBAUFSeEvms9eae3OHSdefPFFUlJSsNvtkgH2wAMPEB0d7ZZwjOjo8dVXX/HRRx9hNBqlILNarezY\nsYPi4mK51ujoaLRaLSdOnJDv8KGHHmLMmDFuVVB2u51Dhw6RnZ3tokRzcnKIjo6WYf7k5GQKCgrY\nuXMnp06dAlSiQmVjxJPQaDTyu//yyy8UFRXh7+//uwbEVYVzHlSv19O5c2fy8vLkuVAURY7ScDZ0\nFEVxKaz+97//zc6dOwkPD2fq1KkA1bpfd4SCAvUSC4uzY8eOpKSkuBQnVhdEQeyf6WAtDkNISAi+\nvr4ef/livsuDDz4o8yVGo1GORoiOjgZUto6wtNxp2Tlf0soQl6VJkyYoioJer/f4fvj7+zNv3jyp\nhJctW8bChQt5++235f6I3omnT592EcodO3akT58+Lm2aqgKDwUDt2rUxmUwyF3ktRqHVamXJkiUA\n7NixA6vVSnR0tCxO7dWr1+86A9wohGe9YcMGCgsLMZvNcpQ7/HaOxT4ajUZ27tyJj4+PnMs2fvx4\n6SW7E+Xl5RQUFMg8a1FRETVr1qRXr160b98eUFmHR44cke2oAJl3rU44zzmz2WwEBwe7vW1Q5a4s\ngwcPpqSkxCXvK7rGVJ7DZjQaWbVqFaD253M4HIwaNYpHH30U8Fy5x5Vwxygos9ksD2d5eTlBQUGY\nTCaOHz8OqIn56oDNZqOiogKNRiNDHd27d7+qgLDZbNIyuXDhAr169XJ7AvlKz1YUhfXr10tLcty4\ncXTs2NGlq4MnxyBcDeJ5ZWVl2O12IiMjq6XKPzAwUHaSmDNnDl988QUbN26UxJuKigosFgt6vV4a\nHgsXLqRTp05ufV8ajYbExESXDhqCqFI5JGOz2di/fz8vv/wyoArg0NBQevToIYWJOxhhoJ4jIUQD\nAwPRarXodDrpGQmjx5muXVhYiMFgoH///pI4ERIS4vYzpdfriY6Opry8XLY28/PzIzk5mSFDhsif\nbd68mW3btlFcXMyoUaMAz5ESrgWhyG02G/7+/iQnJ8vwmSeg0Who3rw548ePl+2zTCYTWVlZ+Pj4\nyPBvcHAw5eXlfPPNNzIEajQa6d+/P+PGjZOyoTr3y0uS8MILL7zw4pbEHeNBBQUFcf78eUANMQgr\nV4Srqgs1a9bE39+fgoICOVl06dKltG3b9ndhLaPRyJgxY2SxY61atXj22WfdZvFeDeXl5UyePJmv\nv/5auuvTpk2TTW1vBaSkpGCz2aql+4eAc+3Y+PHjGT9+vMyJnTlzhkmTJlFRUSGLcit36nYHNBoN\nTzzxBGfOnOHixYuASnQQJAnhsZjNZt5//33mzp0riy99fX3p0KGDy6gPd63POSzbvHlzNm3ahNls\nlj8TTZLht7MXGBhIhw4dmD9/vryHnniXokFtbGwsZ86cAdTasXbt2snpAgCbNm0iMzOTyMhI2bG/\nuspPBITXC2oxdUREBGPHjvV4Z3A/Pz8SExNl/nTv3r2SiCOebbFY+P777+XoGIBWrVoxbNgwjzX3\n/SPcMQrKx8dHXtS0tDQZkhAtOaoLQUFBvPTSS0yePFm2se/Xrx8dO3Zk9OjRUujs3LmT3bt3U1hY\nKF3nefPmuaWJJ7jWxuh0OqxWq7wY//jHP7h8+TK+vr6SJHEzYvHXwqVLl2TS1rktTXVDCLD69esz\nYsQIUlJSZINdT+1XVFQUAQEBMryYlZUlw2eiE8KMGTPYunUrJpNJnp8+ffqwcOFCIiMjPbI2IaAG\nDRrE8ePHOXjwoKwxKioqwmKxoNVqZS74qaee4q9//WuVmY3Xg5iYGCZPnizDnXl5eWzbto3c3Fxp\nuIq716VLF1mLVJ1C12KxcOjQIRYuXAioBmq9evXc0lz4j6DRaAgICKBnz56AqhxTU1M5fPiwfPal\nS5fIzs7Gz8+PgQMHAmq9VJs2bTwyQ+x6cMcoKH9/fzmr6MiRI9Kq8tQY9atBWMBNmzblgQceANTc\nwJYtW9i9e7c8DA6HA51OR5MmTWQn45YtW7rtEIg4d15eHgUFBSxevJg1a9YAajfrkJAQ5syZI2Px\n1W1J/hFCQkLQ6XQYjUbpxbiDKn2j0Ol0dO3a1YUp5gloNBoMBoNLZ+mCggLS0tI4ePAg69evB2Db\ntm3Y7XYiIiKYOHEiAJMmTfIoXVoYCDExMUybNo3Vq1ezZ88eQO3OHxgYyIgRI2TPv7CwMHx8fKpF\nsGm1Wvr37y+NVNH/z7mLRVBQEAMGDGDq1KnVmugHlWmYmZnJI488IhVlVFQUzz77LMHBwdXW5UYQ\nVFq3bk2LFi0oLS2VJQAnTpygZs2adOnShebNmwPIHNXNUE5wBykoPz8/2dtt9erVFBYWEhcX51GK\n8tWg0Wjo0KGDtNy++uortm7dyunTp6UA6dGjB8OGDSMxMdEjJAAh1FetWsX27dtJSUmR4zY6duzI\nW2+9RVJS0k0V+tdCUFAQer2e4uJiOW20bt26N+2iaDQaatSoIXu3efpZOp1OhukCAwOJiYkhPDxc\nUsp9fX3x8/Nj3rx51KlTB/CcR1cZvr6+JCQkMHHiRNmmS7Atq6O7wJUgPIRHHnkEUA3ADz74gMLC\nQtkE9uGHH+b++++/YksvT8B5YvXRo0eZM2cO2dnZkhAxaNAgHnzwwWo1DsX31mg0+Pj4EB4eLhV4\n/fr1MZvNLnVzN+u+Cdw6MR0vvPDCCy+8cILGk+GKP4kqL0T0H+vXrx/FxcU8+eSTzJgxA7h2seyd\nCEH33bhxIytWrCArK0t21Rg5cqTb6Meewt69exk+fDgxMTFyVENsbOxNXbPZbOby5csy4V9dZ0rk\n4iwWi+xYASp5orpCaLcbHA4HNpsNh8MhPZTqzLGKKc27du0C4OWXX+bgwYMYDAY5juSpp55y22Tj\nqkBEmZzLGQSup9P5dfzeDX/BOybEB0hCxNGjR2WX4v81xSQg8gX33Xef7JR8O6FZs2b07NnTRahU\nddRHVaHT6QgNDZUCz9PrcTYeNRoNWq1W5hB8fHxuWuL6doDI5d3s5wsyBvzWrPaf//wn4LkJ0X8W\nzmG/Pxui9fT67ygPygsv7lSIrhEityg6jtwKAs6L2xfO882EEeSBM3XDH+hVUF54cRtANPEUCgpu\nLqvRCy/+BG5YQXlJEl544YUXXtyS8HpQXnjhhRdeeBJeD8oLL7zwwos7C14F5YUXXnjhxS0Jr4Ly\nwgsvvPDiloRXQXnhhRdeeHFLwqugvPDCCy+8uCVxR3WS8OLPQVEU2a1AdFf3Fn7ePhDtfJzHkFRX\nOx+73X5Txp948b+FO4pmLr6L6C3lrbS/MkSX5dTUVDmCIyEhgT59+hAeHu4VPLcgRMV/bm4uABkZ\nGWRnZ5OSkkJ2djag9gqMiori/vvvp3PnzkDV+wXe7PZS1QnnyQe30my06oTVamXfvn18+OGHALRv\n354xY8ZUdTzJ/24nCbvdTlFREYsXL2bTpk0AVFRUYDabiYuLY8CAAQD07duX4OBgj7W2d95Hs9mM\nyWQiJycHUHsDhoWFERcXJwWGr68vRqMRk8nEyZMnAXWibkBAAGazWfYVFGMn3DG/xnmI4YoVK9ix\nY4fcD19fX5KSkhgwYABRUVEA1e5Riaao8JuFXtnIsNvtWK1WFyPE19f3jhaiiqJQXFzM22+/zdmz\nZwG1Lc3gwYNJTEyUAwt/+OEH9u/fj8Ph4JlnngGgRYsWbjM4KvcGvB3grHSutmYxFFN8P9Hj0BPf\n0eFwuEQtKq/tZo0rAXXS9tNPP83q1aupqKgA1JleH330EX379q1K5xJvHZQXXnjhhRd3Fm5bD0qs\n+/z58zzxxBOyg7nzv/v7+8uhb7GxsfTs2ZMpU6bIn7nLUnFuuJiXl8fLL7/MmjVr5IBAjUZDUFAQ\nSUlJ8pn+/v5ERETQpk0bwsPDAbWDd3R0NL6+vpSUlACqZ+Pv709QUFCVww5msxmA3bt3M3PmTLKy\nsqSndvfdd1OjRg0cDgft2rUDoGfPntXmnYjxBN9++y0AkZGRck3ie9vtds6ePcvevXtp06YNoA4x\ndMfeXGtd4tkVFRXYbDa5j1lZWezatYuysjJGjhwJqF6wu7uM22w29u3bx8KFC+X3Hj16NJGRkS4W\nt6IoZGdn8/LLL8vfGzlypFum7DrLCRH2u9W9KJvNxoULFzCZTIB65wwGg0tHepvNxunTp9mzZw8N\nGzYE1GmzoaGhbj1Twgvev3+/nIocERFBQEAAd999N/Xq1QPUjuf+/v5/6PW6M/Qq5OZTTz3F0qVL\nsdls8rN9fX1JTk5m8eLFxMbGAjckN//3xm2I0c7Tp0/n4MGDOBwOOacnKSmJGjVqYDAYOH36NACZ\nmZksWrSIkpISZs2aBUBoaKjbX/LJkyfZuHEjeXl5LocsNjaWWrVqSaVVUVFBREQEDz/8sMsIBa1W\ni8PhkErUXeQFRVHYuXMnAGPHjuXSpUsEBwe7hBsUReHMmTOsXr0agHbt2jF16lTq1avn8byU1Wpl\nxowZrFu3DoDExERee+01EhIS5HfPyMhg3rx5lJaWyjBko0aN/vTeKIoiDQqbzYaiKJSWlsq98PX1\nxWQycezYMf79738D6nstKSnBbrf/LjSj0+lYuHAhAA899BCvvvqqW0cpWK1WNm/eTEhICAMHDgRU\nBV5ZgGq1WiIjI6lXr540cCwWS5UVVOWp1OKsOCsp5+9a2eh1Dmtptdpqye/YbDa2bNnCu+++S1FR\nEaDer+HDh5OcnExoaCgAhYWFfPHFF2zdulXu0z//+U/69evnlrC6OGe//vorL7zwAqmpqXI/DQYD\nfn5+rF+/XirH9u3b06tXL2JiYqplnxwOB4sWLQJgyZIlcr3i2VqtlrS0ND7++GOmTZsm111duC0V\nlN1u56effgJg69at2O12unTpwnvvvQdATEyM9GrEuPBFixbxySefsGLFCmJiYgD1ILojJ+V8ORVF\nISoqCoPBQPv27QFo2bIlPXv2JDo6mvz8fEDNLYWHh2MwGH4nyNxtmTocDs6ePSut/Ly8PMLCwujZ\nsycPPfQQoCr1nJwcTp48Kfds6dKlrFu3js8++4zu3bsDeERRKYrCjh07+OGHH+TFaNGiBeHh4TLn\nBPDSSy+xY8cO7rnnHjnm/Ea8FaPRyI4dOwD48MMPOXv2LLm5uVKoC2ac0WiUF/aP1i/e64oVK4iM\njGTOnDlV3ish1IuKijh48CCBgYFSMV/tO2s0GmrWrMmhQ4cAKCkpqbKyFO9ECFar1YrJZKKiosJl\nf/R6PRaLRRqFe/bswWg0kpOTI6MEDz/8MC1btvSYZy5yJ//617/49NNPMZvNUqC2bt2axo0bEx4e\nLnPBFRUVZGZmSiUGsHLlSnr16lVlBWWxWGREQBhWLVq0kIY0wOXLl9Hr9Vy6dAmAtLQ0SktLCQ0N\nlQrTOYIg/qzRaNziRRUWFvLss8/KzwdVNjVt2hSAuLg4CgoKOHPmjDzj0dHR1eY935YKqqysTE5Z\nraioIDY2loULF0qh5Wx5JCQkADBhwgS2bNnCiRMnOH78OKBeNHcRAfz9/QHo2LEjb7zxBoBkXO3e\nvZvs7GyCg4Olm1yjRo1qC52dOHGCDh06yMsbGRnJF198Qdu2balRowagHs7atWuTkJAgp4D+5z//\n4fz58zz//POMHz8egMcff9xtRBMhgC9dusTkyZMpKCigS5cuADz55JPSwzt8+DAAP/30E/Hx8Uyb\nNk1anDdiZebk5MjvuHXrVhwOB8XFxS6jLIQAcIZGo8HPz08K29q1a5OWlkZxcbEU3uLf3RE6d7a+\nz507x5gxYwgICJBruRI0Gg2NGzeWzD7hvbjjrInPsNvtXLhwgXXr1nHgwAFAFbQ1atTg4MGDUpDZ\n7XY5Y0gohF27djFjxgx69+7tVmPH4XCwa9cuSYoqLCzE4XCg1WrlnZs8eTLNmjVDr9fLd11aWkpp\naSnBwcHSQAGkUXSjMJlMzJ8/n5UrVwLqfX/rrbdISkoiKCgIUA2lY8eO8c0331BeXg5AmzZtCA4O\nvuKkZHezkh0OB2+//bbLd/Xz82PevHmMGjUKgOzsbF5//XWKioo4ceIEADVr1qw2pq+XJOGFF154\n4cUtidvOgxKhu4KCAkC1Ktq1a0dUVJSL+1sZRqOR/Px8HA4HYWFhgHtp1OJzAgICaNm8YN68AAAg\nAElEQVSyJXa7nWPHjgGwceNGli9fzrBhw3juueeuukZ3QljwGRkZJCcnU1ZWJq3Yjz/+mF69ev1u\nDYK4IbyTXr168eSTT3LgwAFmzJgBwNChQ2V+rKoQebtJkyaRlpZG48aNeemllwDkqOz8/HyZM6yo\nqGDGjBk0atSoShZcdHQ0jzzyCAB16tRh06ZNbNiwQXqY8FvuTyAiIoI+ffrQrl07mjVrBsDmzZtZ\ntGgRWq1WepU9e/ZkzJgxbrEwBRlj7dq1+Pj40L9//2vG/0VuyGQyyVCgsNbdAXFeDAYD/v7+nDp1\nioMHDwKqx1FeXk55eblLzkqsV/wsNTWVpUuX0q1bNxl1qCqKiooYNGgQ27dvd5kOq9PpSExM5Pnn\nnwege/fu6PV6FEWR3tKFCxfw8fGhqKhIejEWi6VK+R+Hw8Hq1atZvny53LNHHnmETp06uUQffHx8\niI+Px2w2U7t2bUANQ0ZFRV1RNl0rz3cjsNvtssQFVFn6yCOPMG7cOPmzgwcPymiDyJ9XJ7HutlNQ\nGo0Gg8Eg80gGg0EmuUW4SrxcUZAKMHv2bLKzs4mOjuaJJ56Q/9dTa9Tr9VI4HDt2jOLiYt577z15\nWTwNwVxq27YtxcXFaDQaGabr27fvNS+guAh16tRh2rRpPPLII1J4//vf/+a1116rsoJ1OBysWrUK\ngO+//x69Xs+UKVOkYtLpdJhMJt59910ZRmrXrh29evWqsvAPCAigfv36AMTHxzN8+HBOnjwpz09e\nXh5HjhwhOTlZhoMTExMxGAykp6fz/vvvA7Bu3TrKy8sJDg6WCmHSpEkEBwe7ZX+EQLhw4QLdu3eX\nzxD/XvnPDoeD8vJyioqKaNWqFYBbEv2Vodfr0el0lJaWylCZw+EgMDCQ+vXry5Ba7969qVu3LnPn\nzuXo0aOAqsicDYEbgTBSRTHpc889J5WLUACRkZE8++yzjBw5UpKQRL5Xq9VKhXnx4kXS0tIoKCiQ\n6zp37pzMA93o+lJTUzEajfLZjRs3/t2ZEL9XVFQkyS/R0dHXDKE7v3eTyVQlJW+xWEhLS5OywM/P\njwceeACLxSLTEzNnzuTcuXPUrl2bjh07AtVbxHzbKShQN7J58+YAHDp0iOzsbNauXSuptXFxcZJ9\nIjyWI0eOoNfrueuuu6QQ9LQXU6tWLUBlCxYXF+Pr6+uxQmFnOBwOmfgsKChAo9Hw8MMP8+qrrwJc\n9xo0Gg0tW7bEz89PKvr8/HzsdnuVv4fJZOLFF18EVOu6d+/ePPjgg1L5iNzZZ599Jj2/hQsXusWo\ncL5gPj4++Pj4cPfdd8ufORwO2rdvj8PhkIZQTk4Oy5Yt4/PPP5cJdZvNRkBAAI0aNWLOnDkAtGrV\nyi0X2OFwSA/cZrMRExODoigyX6AoCjabjbKyMsrKygA1N3vhwgUURZFRAk+ccWH8iTwXqJ7a448/\nzt///neZJwP1vGi1WqnI/Pz86Nu3b5U6XBw4cIAJEyZIw0V8tr+/vzQA//GPfxAQEOBizIh122w2\n+Y5SUlI4ceKES/7RZrNJhXcj0Gg0jB07lpycHEl+OHnyJA0bNnTJ31y+fJmvv/6aiIgIkpOTgT9H\nQqoqO1Or1RIQECCNGB8fHz766CPOnz/P2rVrAfjll1/QarV069ZNKluvgvoD6PV6GaIxm82cP3+e\nLVu2SIuqvLyc0tJSioqK5IXW6XT4+vpSq1YtaXV4uo2LOGw9evRg6dKlJCcnV4t7fPnyZdnCCNRq\n8I8//lgqlSt9b5FMr3z4AgMDCQsLk4lvHx8fbDZblRSUw+Hg9OnT8jPDwsKYNGkSBoNB7o/JZGLt\n2rVUVFTIhG2TJk1u+Jl/BiI8VFhYyIoVKwCVgnv8+HHpmQIEBwfTvXt3xo4dS9euXYHrV/5/BJvN\nJp9VUVHBpUuXOHfunLTyN2/eTGFhIWVlZRw5cgRQPcP4+HgSEhIkOSgqKsqtRAkBQdUXobL4+HjG\njRsnvVBQvaXVq1dLkguo73DgwIE37AUrisKyZcs4dOiQi1LR6XR8+eWXPPDAA8BvhIIrnXOr1cq5\nc+cAlXhjtVpdzn5VPTyNRkN0dDSzZ8+WbOPIyEiys7PJzs6WRs8PP/zA+fPn6dixo0c83T+C1Wql\nXbt20hCyWCwcO3aMw4cPSw/KbrcTHh7O5MmT3VJP92fhJUl44YUXXnhxS+K29KBErQfA+PHjWbZs\nGYsWLeLUqVPAb8l3h8MhQwnC3U9LS8NoNAJVd5GvBUVRZAKyoqKC2rVrU6dOHbZt2wZA165dPRbu\nu3DhgrRshTUHrrmKyt0AruQ9gWqZNmjQQFqcu3fvJjc3V1L6bwQVFRV88cUXkmxx9913ExYWhslk\nkrmB3NxcLl++TKNGjSRBo7pCCw6Hg5ycHJ5++mk2b94MqB6dKNIV64iLi2PUqFF0797dI/nMjIwM\nQCVoVFRUsGPHDpkTE4QARVFkiK9+/frk5ORw5swZWYs0ffp06tSpg06nk16LO/YxKCiI2NhY2W/y\n4YcflpEJ8Q737NnDCy+8gNlslvd1xowZMvx4I9BqtbRo0UISSASaNGlCv3795He8krcoupUcP35c\nknGKi4vlXRDeX79+/WRnhxuFkFFDhgwBVG/FbDZjNBqlXNiyZQuFhYXodDq5Z4qiXJVO7s5eiA6H\nA6PRSF5envToRDPikpISGXny8fGhW7duN1QQ7w7clgoKfntBiqJw/vx5MjMz5Qv08fHB19eX+Ph4\nOnToAKgvZMOGDZw8eVImbO+55x6PbLrFYmH58uXMnTsXUNlw77zzDu+//74MTU6cOJGpU6d6pJ5g\n7dq1LrUNZWVlnDhxQhbfiVi8n5+fS8X4lSDYbGJvRS3QjUJRFC5dukR6eroU6kVFRTz//POUlZXR\nqFEjQE0Wnz17lsDAQLm26uqsXVFRwaOPPsqOHTuuGZLNz8/nwIEDstjZnSguLiY1NRVQjYSgoCAX\nxWMwGLj77rtJSEiQe5acnExmZibLly/nwoULAHz22WcMHz6cxMREt7HmNBoN4eHhTJs2TRY89+3b\nF41Gg6IoMjz0yiuvUFRURFBQkAzT3nPPPVVWkKK+Srwbg8HARx995HKXnM+Kc7uq/Px81qxZwy+/\n/AIgO2L4+/vTv39/QC0Id4fB4XxXDAYDWq2W/Px8md85cuQIZrOZkpISGWYT7ygmJkYacFczZG/k\nLjg3jN61axfl5eXSYIiJieHChQscOXJEsiH1ej2jR4+ultz5lXDbKijhBX344YesWrVKdi8Htd3M\niBEjiI+Pl4c2LS2NzMxMMjMz5cXv0KGD27qEixdvNpv58ssvefHFF+WLnz59OqGhoWzcuFG28nnn\nnXfo0aMHd999t9uVVHx8vPQczWYzFouF9PR0SSzR6/Xo9frrEhQWi4VTp07JNcbHx8vu2TcCu93O\nxYsXXSr8z5w5Q05ODjabjV9//RVQL5/JZMLX15cJEyYAKqNI9CV0Zmzq9Xq3WJfiMwoKCigtLSUk\nJMTluwqGnLi8ubm55OXlud2zczgcHDhwgB9//BFQizyTkpJcKvjj4uIYPHgwMTExLsI0JiaGpKQk\nPv74Y0Clda9fv55Ro0bJs+6O9RoMBlq1akVSUpL8TJvNhtVqlexMQU8ODQ2V/R3d8ewRI0YwdepU\nKTRHjx5NfHy8yz10VmDCO7HZbKSkpLB8+XLJkBRratiwIfPnzweo0vm+GjQaDXl5eUyYMIHdu3cD\nagSnd+/eREREsGTJEkAl45SUlBAeHi4VZu/evWW/SWF4ms3mG2IZOncn2bNnDwaDgRYtWgBqL76T\nJ0+yZMkSmTszm81s3ryZPn36eD2o64Xdbpehl8WLF6PT6Rg0aBALFiwA1OS1oJo7j2VISEggKirK\npTbEHVa5EKagjtZYsmQJTZo04c033wRUNp9Go2HatGls3LgRUFvQvPXWWyxYsEDWQLgLf/nLX+Re\nZGZmEhgYiJ+fnxRkf6Y90KlTp0hPT5cGQWJiYpWsKa1WS1xcHKNHj5b9FMPCwjAajWRkZMjnfPLJ\nJ1y6dEn2JwSV1m02m2nVqpW8VLGxsW7vlBAXF8e2bdsoLCyU4eKysjJWrlzJO++849IpoaoJ9avh\nyJEjLqFoPz8/+vTpIyncISEhsttA5e8eGhrK0KFDAViwYIFsX+VMU68qRLmH8BAKCgq4dOkSJ06c\nkH0JS0tL0Wq1BAYGupUc5O/vz86dO6VBKmjvlfdC0NGFDPjpp5948cUXSUtLc2nRFBAQwLPPPis/\nzxNQFIWNGzdy4MABuZ6kpCTGjh1LUlKSDKGvWLGCoqIiLl68yJ49e+TvNW7cGJ1OJ0Obfn5+f/rc\nOxwOWT/65Zdf8v3339OsWTMefvhhQA0RJyUl0bBhQ7Zv3w6ocurTTz9lwoQJVQrr3yi8JAkvvPDC\nCy9uSdyWHlRubi4fffSR/PugQYOYNWuWtOacY8/CUkpPT+fEiRPEx8fLcIM7Q2vis4R1PWHCBEn1\nFR5AUFAQn3/+OQCrVq3i9OnTrFu3jjFjxrh1PYGBgXz66aeASiIpLi4mOztbWm5/ZHU5hysXLFiA\nxWKRSePZs2e7xYOKi4tzSdqLNTl7ort376Z3796yli02Nhaj0egywNETYQdRH+Lv7+8Ssx86dCif\nf/65fMcOh+OGizn/CM7NVH18fOjXrx9NmjSRXvC1QmVarVYSY+rVq8fZs2exWq1u3yuNRiNDyWFh\nYZjNZrZv3y7JHaCGAnv27Env3r3l392xDpF3qwxnT81qtWKxWKR3snjxYo4dOyZzWKCGTwcOHMjw\n4cM9SsIRoe369evLETfPP/+8rJsT6YCoqCj++9//UlZWxj333AOo5154iSL8eKN7KLzyDRs2cP78\nebKysmTY8LXXXiM2NpY6derQs2dPAFavXk1RURGTJ09m2bJlgGcaRl8Nt52CstvtLFmyRLZYcebo\nX6nmQbCe3nvvPU6ePEloaKhkFLmz1ZF4aaGhodStW5e6dev+rkhQxLoBhgwZwpo1a2SiFHApcKwK\nNBqNJIeMGzeOl19+mSVLljBo0CAAOUfoSnA4HJw/fx5QL9C6deswGAwypxEWFlalPXMWalf6HHFZ\nzpw5Q1BQEFOmTCExMRFQ91jMrHL+vOqIjdvtds6fPy+Vk3i2GI/iTjgcDuLj44mIiADUEE+DBg2u\n2ED0jyAao3qqYFcYK4K1lpub6zJaIyYmhueee04KYHcpgSvddWeIfFhpaSn79u37//bOPDyq6nz8\nnztLtglJSMgeFtlRsIigAm6AWmVTBEGsCKLVx1JafdQKtvi0SuvyiGxqZXFBrVZcaQFpBSuKiMgi\niwkKJCQhIWL2hSwzc+/vj/md1wmgIt4ZBr/n8xfCmHty7pnz7u8LBAp8laBW37XLL7+cefPmhawO\nSa3L5/Nx1VVXkZ2dzTnnnANA9+7dRTlT+9jY2EhsbCwZGRnyufT0dFsaxQZnP0dHR0tX+pUrV8pn\nfvWrXzFw4EARUKtXr8ayLPbs2SPKvhZQ34NpmuTn50tsICcnp1VxoMLn81FcXCwdFXbv3o3T6eS2\n226Tz9v5pVUvrVevXgwaNIj//e9/oikFr099QZ1OJwcPHmTPnj1iNdgloIKfM3HiRJYuXcrBgwel\n28HIkSPp1q0bSUlJrZINvF4vc+bM4ZFHHgECVkNcXBxXXXWVjA6xM9ZzPEpLS4GAlXzOOee0srTU\n/xeuLvB+v180zj179jB79uxWhbpRUVF07949JJbJ4MGDmTBhAgC9e/f+UReo6pYAAas9Pj6ezMzM\nkFoI6mcfPHhQ/i4hIYHx48eTmpoa8hIBlRQRXEpRX1/Ppk2bxNuizlZ0dLRYYA8//LCt/QqPJlhA\nNTU1kZWVJW22XC4XXq8Xv98vHSe2b99Ohw4dGDduXCtF2i7UOVqwYAEXX3wxhw8flrOyfv16srOz\nSUtLE+9PSkoKFRUV4oUB5F4LB6edgHI4HPTu3ZtVq1YBga4Jubm5EjSHQIbKW2+9xbx588TdEBMT\nw/Tp07n88stD+mVJSEhgyJAhLFq0SLoQTJw4UdJMVZDyueeek3RTdTjVGAe71/POO+9w1113iYm+\nbNkyHA4HaWlpkv6bnJwsFfrqUk5ISGDmzJn87ne/C0uaqWmaLFu2DAgE2KOjo8M6HE2hakS2bdsm\nw9wKCgrIzc1tZY0kJycft+nuT8UwDNq2bcutt94KIFq/1+ttZbF813NbWlqke8OOHTuIjo7G5/O1\nynALBaWlpa1Sq3v27MmNN94Y1hRlpeXX1tby0Ucf8fzzz0vKvd/vJzo6msTERMaPHw8EmhKHcn1q\nz48cOUJTUxMJCQlSt3b48GHKysr4+OOPJauwX79+jB8/nri4uJDcU+rdd+jQgc2bNzNz5kwZZJqV\nlUVxcTErVqwQA0B9vrKyUtpIvfzyy2F7pzpJQqPRaDQRyWlpQQ0fPlwaRb7//vuMHj2auLg48XMf\nOnRImj127doVgEWLFnH++eeH3H9qGAadO3fm3nvvlc4WmzdvJj09nfj4eNHIlyxZQl1dHd27d5em\nsqFaT05ODq+88ooMUpw7dy6madLQ0CA1KwkJCVRXV5OYmChNd1955RUyMzPD5nP2+/0yCdayLPr3\n739SE3PtoL6+njvuuEMC7MoVA98WTl522WV07tw5JOtzOp3SnFM1ia2urhbrJCYmBqfTeYyWrYLx\nKk24tLSU0aNHk5qaGvI+kMXFxTQ3N4vbuHv37rRv3z68zUX//7tJSkqif//+fPnllxw4cAAIpKdH\nR0dz9dVXy0SDk4nr/RjU7x4XF0dJSQmbN2+W2OK+fftoaWlh4MCBTJ06FQi4z04kkeSnllaoDjNL\nly6VrjOqVKCqqkq6XcTHx1NWVobf7+eDDz6Qz9lZsvB9nHYCyjAMsrOz5YWWl5ezYcMGDh06JD5m\nl8tFamoqY8aMYcaMGcD3JwbYjcPhIDU1VSrBq6ur+eqrr9iwYQPPP/88EMhWS0pKYsaMGceNodmN\n2+3m3nvvBWDatGlUVFRQXFwsSQkHDhygU6dOUhAK9jU+PVHq6urYv38/ENjD4E4X4US5z1wulySw\nmKYpl4JSKB588MGQuiDVeVUTaRMTE0VIqg4IwZ9TjVDj4uIkZjhs2DCJ44Xy/FuWxb59+ygpKZHn\njBo1KqTxnaMJdnuqe+K2224TZbWwsJAzzjiDadOmifAP9Z2gfn5sbCw5OTls3LhROtm43W7Gjx/P\nlVdeKft0oufdrnU7HA7JRE1MTKRjx440NTWJUqpCE36/X9x+BQUFpKSk2D7h93icdgIKAhenylJb\nuHAhmzdvZuPGjRLsu/DCCznvvPPIzs4+ZS06DMOQgGRqaiputxuXyyVzXw4ePMgFF1zAuHHjwrbG\n4NRaj8cjY0ciBcuypNtFUVERI0eOPCXWEwQulF69ekkWqOqRlpSUxMMPPwwEEnTCsT518Z5IokRs\nbKxkgUH42kOpZ2dkZIgAV1lopwLVkT4hIYHp06cDgbhUamoqbdq0aRXLU9jZ6+5oXC4XvXv3pqqq\nSu6ps88++6SzM0OB2jOPx8Mvf/lLIDCA85///Ce7du2S99qnT5+wnSsjnNMRf4CIWUgoOLrtSrB2\nrAnQ0tJCVVUVEHCJxMfHn7IvbktLCytWrODuu+8GkPqr0aNHS4cQu3rb/RywLIvi4mLeeustyT4b\nOXKkLcMbfwqmacp3zrKsU+YyVpOOVaIGcMLtxn4GnPSG/5/YHY1Go9GcfmgLSqP5Dvx+v4yTeO21\n1xg+fDjt27eXerVIcMtEEk1NTXz++edSWK2GJf4fsRI0381Jf1G0gNJoNLYRzpiX5rRBu/g0Gs2p\nRwsnjZ1oAaXRaDSaiEQLKI1Go9FEJFpAaTQajSYi0QJKo9FoNBHJadlJQqPRnF4Ej8EI1wwvzemP\nFlAajcY2gmcywbcDKI8cOQJAfn4+nTp1wuPxSONbu5+vnut0OsM6XE9jP9rFp9FoNJqIRFtQGo3G\nVoKtmMbGRsrLy6XHYkJCAvHx8SFrkKwm+q5atQqv18vYsWOlce7JdLRQffxO9v//PkLZnPbnws9S\nQKmxA/X19dJqX42+DvXYge9az/GmmYZrHaZp0tjYKJM8q6qqKCsrIyUlhezsbCBwcZyqzu8Kn88n\n78uyLJqbm6mqqpJ1WZZFSkoKiYmJIXXdqAajXq+Xzz//HIBPP/2U8vJy8vPzxTU1YcIEhgwZQlRU\nVMjW8kPrbG5uFjdaVFQU0dHRp/yyU2e9qamJwsJCioqKZEpzp06daG5uxuVy2T7h1zRNdu3aBcDj\njz+O3+/H6XRy++23n/TPtFMoWZZFbW0tjz/+OBBon6VGBKnJAkuWLGHgwIER0x5KTWJWnc4hvML0\nZyWglLZTU1NDbm4ua9askflCOTk53HfffSEZq368dTQ2NpKXlwfAhx9+SGlpKSUlJdTV1QGBUQTT\np08nNTU1pC+8ubmZoqIiFi5cKOvJzc2lqamJxMREcnJyAJg5cyZDhw49oZEOocDn87Fz507mz58P\nBIbf+Xw+SkpKZK5Wamoqffr04Z577hGt2M69U7OWGhoa2LZtGy+++CLr168HAmeqoaFBRiUALF++\nnDFjxvDss8+GXUh5vV42bdrEggUL2Lt3LwBt27Zl8eLFdO3a1dZ9+THti0zTlDNeWFhIRUVFq/EW\nXq+X0tJSOnToYHs3eNM0ZdBedXU1ycnJIR0G+mMwTZP8/HzuuOMOGbHe3Nwse1tcXAzAY489xoIF\nC+jQoUNYBEGwFaf+7Pf7+eabbwCYPXs2u3fvpkePHsycOROA7OzssI0I+dkIKMuyOHz4MABPPPEE\nmzZtoqysTKbslpeX8+yzz/Lb3/5Wmn2GAtM0qaqq4v7775epprW1tcTFxVFVVUVTUxMAH3/8Mbt3\n7+aFF16QC9iuF25ZFl9//TUQmCS8Zs0a3G63TM5MTEzE4/Hg8/nYt28fANOnT+e2227jzjvvDPtl\n6/f7+eijj7jhhhuorKyU30FZBMo9VFxcTHl5uUyIBfuGKiqlAmD37t089NBDbN68WQYWOp1OvF5v\nK2u4sbGRN954gyFDhjB58mT5XChR1tKiRYt4+umnOXTokAhWt9vN3LlzeeKJJ2xNQDjRc+n1eikq\nKuKVV14BAlZBcnIyPXv2bGUFV1dXU1ZWJlZD8J791Cmx8fHxAKSnpxMbG4vL5RLFNdgqOdp1F6rL\nNtjd+dJLL/HZZ5/J4D81iDIpKUmGBrZt25adO3eSlZVl+zDMYGHk8/nw+/00NTVRXV0NBPakoqKC\nd955h9dffx0IuEz9fj/79+8XZXbq1KlkZmbicDhCbulFhh2p0Wg0Gs1RRLQFdaKuBcuyaGpq4qWX\nXgJg5cqVdOzYkeuuu04mQ37++eds3LiRHTt2yDTeUEh/v9/PwoULWblypVgsycnJ9OrVi8TERLZv\n3w4E3B9bt25lzZo1XHvttQC2aUzNzc08+eSTALz55pvy/LPOOgsITMSsra3lX//6F++99x4AX3/9\nNfPmzaNz586ynlBrR0qL3bt3L+PHj6eyslK0PKfTSVpaGikpKZSXlwNQUVHB4cOHKSgoYPDgwbat\nw7IsfD6fxOiWLl3K5s2baWhoaHX+oqKi8Hg8ssba2lp8Ph/333+/xPKuuOKKkGnjpmmydOlSAB5+\n+GGioqK46aabqKmpAeC9997jgw8+IDc3V6bZhite4PP52Lt3L3/729+oqKgAoH379gwYMIA2bdrI\nOvx+P5Zl4XQ6j3Ev2bFWdWbT0tLIzMzkzDPPlH8zTZOWlhZM0xSr0+l0EhMT84NWlM/nO6n4tfod\nCwoKePXVV6mvrxcPRfv27bn++uu54IIL8Hg8AHzzzTd4PB6am5vFsrTre6hilgBlZWXs2rULt9vd\nyuW4du1a8vLyxJUd/K6KioqAwDtU+6EI1V0R0QLqRA+Dz+dj165dvPvuu0Bgo0eMGMGUKVNk41JS\nUsjLyxPzOlQos7mxsVHiORMmTGDy5MlERUXxxRdfAPDoo49y6NAhVq9ezfDhwwF7BJS6bFWM5qKL\nLuKWW27hzDPPbOXaNE2T3r17S33Ke++9R2NjI8uXL+fyyy8HCOk0VBUwBrj11ltFOKnnZWZmMn36\ndBISEnjmmWeAQFzB5/PZ7v9WhaMqdvLJJ5/Q2NjYKjAcFxfHkCFDuOWWWzh06BAATz31FF999RVV\nVVUyZXfo0KG2u2YUBw4cYNasWfLfs2bN4vrrrxeBsG3bNg4cOMBTTz3FokWLAPtcoN+Fuug3btzI\nzJkz+eabb8TdOWbMGOLj449xrSm3e/A7tOOCMwxDZlFlZGSQnJxMTU0NaWlpQMDNNnfuXM4991z6\n9OkDQFZW1gm5+E52H5WA2rZtG5WVlbjdbomDX3DBBUydOpX09HTZx9LS0mPWY+cIExVieOeddygp\nKWkVDnA6nRQWFkpiBAT2NDo6mksuuYSxY8cCgbs02GUL354D9XPsIqIF1IlgWRaNjY289NJLIuGH\nDx/OLbfcQkxMjGxcTEwM0dHRxMTEtNLc7MbtdnPxxRfz2WefiY/91ltvJTs7G4fDIS+2V69eFBQU\nyMEFbBlxbhgGMTEx3HDDDUBAOCYkJOB2u1tdAg6Hg3bt2nHeeecBUFJSQlFREeXl5axduxaAyy67\njNjY2FYXrl1fFL/fLxbvtm3bME0TwzDEF//AAw/Qp08fXnjhBYmTeb1e2rdvT7du3WwXnIZhiPJQ\nVFQkmmNiYiIAgwYN4sEHH6RLly6iXXo8HmbMmEFZWRlbtmwBAoWoPXr0sHVtELjYFyxYIEJ02LBh\nTJ48mejoaFEyvF4vzc3NbN26VWJVoRRQpmmyceNGACZOnEhLSwu33347d955JxAQ6ke/J8MwcLvd\neL1e+Te73qVhGHTq1AkIWFAul4uDBw+KUvrcc8+xdu1aHA4Hl156KUDIg/3qZxihD/EAABOfSURB\nVHs8HnJycqiurmbYsGFAQEC1tLS0GuqYlpYmsSm79wcQa/ubb76Rc64EuIpnBj8vJiaGkSNH8vjj\nj5OSkgK03rOjO4TYbUmdtgJKuYfq6+tZtGgRq1evpl+/fgA88sgjxMTEtNqwgoICPB4P2dnZIT2Q\nTqeTCy+8kPj4eBFQmZmZEqxV2lOXLl3wer0cOXLEdqvO6XTKRQ/fWgjBqIOlMqni4+NJT0+ntraW\nF198EQhYEldeeSXnn3++fE5pdz9lDy3Lor6+nrlz5wLfanVut5srr7wSCGTsLV68mBUrVoil5XA4\nZB3HC3z/FIIzwCCwZ5mZmYwePRoIKBldunQhKipKNMT09HSam5vx+/2yxn/84x/8+c9/tv2LWl9f\nz/Lly8U99MADD4iFrlyT6owdOXJEEj7U98BuLMuitLSUSZMmAVBXV8f48eP54x//KAkax3uuUigr\nKytF+NuFYRjiKqurq2PXrl3k5eVJKvfXX39NUlISN998M23atAFO/PyYpnlS71TtweDBg5k2bRqW\nZcn+FBYWMnfuXM4//3zxosTExBAbGxuSchjDMOT74/V62bt3LxkZGeKeTk5OJiMjQ2rJAC655BJm\nz55Nenr6MZZRsKKv7gWfz2erUqSTJDQajUYTkZyWFpTf7xe/6dKlS1m2bBkxMTHin1cmqGVZYp1s\n3bqVlpaWsBQytmnThkGDBslzlOblcDhEC/nPf/6D3+8nMzNTTGw7XR3Hs5iCCyiLi4t5+eWXpbCx\nqqqKtm3bYpqmpJ1+8skndOjQgXPOOcfWtGXTNFm9ejVlZWXyd06nk+7du3PBBRcAsGHDBlasWEFV\nVZVYS4ZhUFRUxBtvvEGvXr2AgBvJDmvF6/WK5miaprj3lHaZkZGB2+0WCwACpQINDQ1YliWuZBWE\ntpstW7ZQXV1Nx44dAejbty+GYWCappRXFBcX43A4yMjICGlRpdqDu+66SwqrJ06cyLx5837QYvP5\nfBQUFNhuPSmUVVlTU0NRURFffvllq3hK//79SU1N/dFn5mTPmNqLhIQEzjjjDLZu3cqOHTuAwBmv\nqqri3XffFet9xowZIUt7NwxD4n+dOnUiPj6+VcJI586daWhooLa2VqygSZMmkZOT0+r3/74Qid0u\n5dNOQHm9XioqKnjssceAwCVRX19Pnz595NCrC8OyLAlo5+XlkZCQcFy/uN0YhnHcF2WaJhs2bABg\n+/bt+Hw++vfvL26JUGKapsToHn74Yd5///1WWXMej4e4uDh69eolRXo9evTg6quvpk2bNq0uvJ+6\nfz6fj1WrVskXw+Fw0KZNG8466yx27twJBIK41dXVrQLEhmFQX1/P6tWrJc4zYcIEW96pYRh89dVX\nsj7TNCkoKJD3dc4555CcnExhYSFvvfUWADt27MDv9+NwOORiTE9P/0nr+C62bt2KYRi0b98eCOyZ\nqt267777ACRDbMKECSE9U16vlyeffJLPPvuMvn37AjBnzpzvFU7qnDU0NLB7926GDBli+7qClbCs\nrCycTicOh0MUCofDQadOnU5JA1nLsli1ahVvvvmmuINVF5Di4mIWLlwIBJTre+65x5Z49NGo+B/A\nqFGjKCgoIDc3V4qEGxoaKCwspKGhgXbt2gFISOLojMujBXao7tTTSkCpDLU9e/aIBeXz+cjKyiIt\nLY1PPvkEgI4dO9KhQwd8Ph+ffvopEPD3jh49OiQdlE907UeOHOGuu+4CAt2dExMTmTJlSti+MOog\nFhUVUVZWRktLiwjShIQEYmNjMQxDBH16ejoejweXy2VrwLahoUE6fACS2VRcXExubi4QiCGoxAn1\nZTAMA7/fz6FDh6QF0YQJE2zJcnK5XGKVrVu3Ti5/dX72799PRkYGNTU1YrG4XC48Hg+1tbXyDoNb\n+NiFZVnU1dXhdrulqLukpIQjR44wf/58Nm/eLJ/LysriuuuuC8mZUr/X9u3bef7550lISJBsQY/H\n873vQCkjubm57NixgwsvvDAkl5qKsYwbN47i4mI+/fRTsWpdLhexsbGYpnlSbZZU66Qfg3qOSvFW\ncWkInPsDBw5QUVEhcdjHH3+cjRs38tprrx032/Gnon5WWloaHTt2ZP369VLGUV9fT2NjIz6fT4T6\n9u3b6dq1K9nZ2SLcjmfh2ZlpGMxpJaBUEC41NVU0sNGjRxMfH8/u3btFM9m3bx8FBQWkp6ezdetW\nILD5/fr1OyV9ylRvwKeffpoDBw7I348YMYJevXqFZU0Oh4PevXsDgTqovXv3Ul1dLRr5hAkTSExM\nZN++fVKr5fP5GDNmjK3zeyzLoqamhqqqqlYXeXV1NaWlpeKSDb5Agq1Rh8NBTEwMgwYNAgIapx0u\nPrfbLen1S5YskUtNZcjV1dXRu3dvrrrqKlGE4uLi2LlzJw0NDeKGbGpqEhehnXTr1g2XyyXPvuOO\nOzAMg08++aRVTc+yZctISUkJmWsPAh00PB4P9913n7gcv+95wW7RZ599loMHD7aqjbILwzAkiaR3\n796MHTsWv98v51ldrOXl5eJWPzq79Yd+/smsST3nr3/9K06n85hONk1NTSxevBiA+fPn8+mnn3LT\nTTeJpW5nZxe1nqioKPr378+LL74oCpeqEXM4HJIFmp+fLwkbwYrid/1cu9FJEhqNRqOJSE4rCwoC\nJnzXrl3p2rUr8K3kvuiii0STLCkpobq6mj179kitTUNDgwS07e6i/F2o56hGqM8884xYCLGxscye\nPTtszVmD+5TdcccdDB48mNraWkmjjomJoaWlhbfffpuVK1cC33ajzszMtG0dqhdbfX19q35odXV1\nrRqxqjUHa25t2rShffv2jBgxglGjRgH2BWUNw5DamJ49e0pNlDoj3bp144EHHiAzM5ORI0cCgbTl\np556iuLiYrG4lEvQTgzD4Nprr2XLli2S1GKaprhm1BpTUlLo27dvyFzG6n0VFRXRt29fhg0bdkLP\nMk1TYsGff/45SUlJP+gSPFnUWUlMTGTUqFE4nU7Zp9raWsrKynjzzTcZOHAgEKhHPLqY+Lt+h5+y\nXpfLJS67o4mPjxfX/6RJkxg8eDAbN27kv//9LxDwtITC2szJyZFYE9AqGUmttW/fvqSnp4etOezR\nnHYCyuVyHbdGILiYNCkpCZ/Px4EDB6RGRLU5CWWR7tEoM7moqIjZs2dTXl4uX4Srr75a3GvhQl3m\nWVlZUpsVfMG7XC6io6PFH15bW0tTU5OtB9Pv97Nu3Tpx+ai/UwRnPKo1qS/L1KlTufHGG8nJyZFY\ng51rUz9zxIgR7Nu3D7/fT8+ePYFA5/KEhAQsyxJBf8YZZ7BlyxbJyASks7jdxMXFMWvWLInR+f1+\n1qxZw7Jly2Qvp0yZYnuH8GDUc3w+H0OHDj2hRAzV3fzVV18FAokBY8aMkTokuzm6A4Lb7ZaaQJ/P\nR1xcHAUFBXz55ZcA3H777fTo0eMH9+14iQF2EqxkjB8/nnnz5nH33XcDcOWVV4ak4Lpt27Z07NhR\nlGTVXT06OlqKiUeMGHHCmc+hiEOddgIKftgHqrJ3unbtKofK7XZLB95waAKWZYnmlpeXJ10t1GG4\n+eabT9nMF7fbfdyqb8uyWL9+vXQPj4+P5xe/+IWtzzYMg7S0NOLi4qQrgorZxMfHy6Wnik2joqK4\n5JJLAPj9738vs6BClYYLgRRcFZt49NFHgcCXWfUkC66iv+iii+TPilC8V9VlQ2XNGUagNdPixYsl\npjFhwoSQninV8WTDhg0MGzbsOwtxFao79qOPPir93gYPHsyvf/3rkKxTzfACZB5csOYfFRVFv379\naG5ulvR4Nfrjh9Ljw5XIZBgGFRUVmKZ5jEfB7uc4HI5W3zmfz4fb7eaKK65g2rRpQKDJwKmwnBSn\npYA6EQzDaBWMjImJEWsq3HTr1o3o6GicTqdYesFNLMOBGsKn/nx0vzjLsti/fz/vvvuufG7o0KG2\n16u4XC5Gjx7NypUrpVGt1+slNTWVSy+9VCyEffv2odo2nX322UDgHYZy4KS6EPLz82nTpg2pqamc\ne+65sm5orQhZliV9+9TFfOaZZ4ZsfU6nUwLmPp+PlStXSvsnCNSxhBL1O37xxRfExcVxzTXXHHOO\nVC9KgE2bNjFv3jz2798vbuI5c+aExMpTGb5K8DQ1NWFZFhkZGSK0OnToQHZ2NjU1NZLRWlJSEtLx\nMic6NVd9rqqqitdffx2/3y/jLUKldDidTi666CK2bdsGBEYS9ezZk1mzZtGtWzeAY3rufd/vERKX\nre0/UaPRaDQaG/jZWlAKFRtoamoKe4GeilUcPnxY3Fhq1EdwcDLUqI4awenIwf8GgeaRkyZNoqam\nRmo1Fi5cGJI9S0xMZN68eVJD09DQQGpqKvv37xctTK3V6/WKFhfqLiBK89+wYQNNTU107tz5e5NY\n6urq+Pvf/y5+e0DS8kNBcKFlSUkJW7ZsISoqSrqHhyquo1A1WGlpaWzZsoXVq1dLrAICIxzefvtt\ndu/eDQRc29HR0Vx//fXcc889ACErIFbF+craiIuLwzRN0tPTxS1aU1NDQUEBlZWV5OfnA4EGqcOG\nDZMu6HajkmcqKytJTk4+pizCsiyam5vZtGkTADfccANVVVV4PB6WLFkChM5lDIHvvWogm5CQQKdO\nnSQ+rT6n6saUZ0V5MYLPuS7UPQn27Nkj5r3T6aS4uDhsSRKqmSbAmjVrqKiooHv37kydOhWwb/bT\nieDz+airq5MMwqSkJPx+P6ZpSueGa665hsrKSlJTU1mzZg3Ad2Yd/VRUI1Y1Qrq6upodO3awfv16\nCgsLgW8D8m63W9xsoRROlmVJ8e++ffuora0lPz9fumqodi/qQoFAw9aDBw/icrlE8Rg5cmTI1wmB\nprSFhYUkJSXJGIRQxzSVAJwyZQrz58/noYcekjEjTqeTkpISfD6fnJuxY8fym9/8hrZt24Z8bSop\nQj1HzStyuVwioPLy8mhsbKS5uVnaeWVnZ8t3wW5lzLIsOc9/+tOfcLvd3HzzzZJ44/V6WbNmDXPm\nzJE2Wz6fj+joaF577TVxs4UKp9OJx+MR5aq6upq4uLhjFEFVf6oUOJV8Eo7Y1M9aQOXl5YnUD+44\nHS7+/e9/A/DBBx9QXV3NiBEjpFt3uAKPKnsxPz9f2vZ89tln5OTk8OWXX8pY+paWFjp27MjatWvJ\nysoK+RrVhQKBdir19fUyeFIRHR1N37596dKlS8jXE6y41NbW0tLSwu7du/nDH/4ABGYvdejQgaqq\nKubPnw8E2jG5XC4GDBggWWqhblullIy1a9diGAbDhg2TYtlQo5Sqa665hkGDBvHkk09KESwEEiAm\nT54swjpUqeTHQ2n0Kp7kcrnw+Xw4HA4RUIcPH+bcc8+lurpa9rFz584ha08VnFCzbds2Dh06xPLl\ny1sppyo2rO6ptLQ0SYMP9d45nU4GDhwoZSW5ublkZma2Ss5Qlmlwzz6VjBIcDw7VWJefrYCyLIsP\nPvjgmJqncGakKNN5586dGIbBWWeddUpaLZWWlrJ06VJJSjh8+LDsi0okGT9+PE8++aS4JcNB8OEe\nOHAgGzdulBZIFRUVpKSkMGzYsJC0fDneWpSbp0uXLuTl5eH3+1mxYgUQGMiXlpZGeXm5aNrKunv6\n6adDdskdjXpvFRUVMqsn1EMJj8bpdJKZmcns2bNbrcvOjiMni3q+mr1mWZa408eNG0dKSor0wISA\nVRgXFxcSV7bD4ZDkkAEDBrBixQpM0xQlzDAMyV69+OKLgUDz63bt2oVtH6OiokQBbGxspLi4mIMH\nD0rCTUtLi9SQKo9Qnz59pNepUggSEhJCYiXrJAmNRqPRRCQ/WwsKAtqRSmkdMmQIo0ePDptmYlmW\njGpQAUbV/yucGIZBdnY2ffv25cMPPwQCCRqxsbGcf/75/PGPfwQCqfCnqi7L4XCQlpbG5MmTxUIo\nLCwkPT2dG2+8MSxWp4qLATzzzDP85S9/YefOnVL7U1FRwZEjR8jKypLEgC5dujBgwAB69OgRtnOl\nNP2+fftSWVkp2u+pIBxB8pNFWSdOp/O450clfMCx4yNUtxk7vg/KI/H000+TnZ3NunXr5ExlZmYy\nZcoUJk6cKPG9cCZyGUZggKEqU6itreWLL75g+fLl0i3FMAyqqqqOiUuZpklDQ4N4DtSe2d7xIpyd\nFX4A2xeybt06FixYAAS6BHfu3DlsB8Dv98t4izlz5tCvXz8mTZp03HqacOD1eiXxQLk0Iu1SgW/b\nrahzeSpGIwQT3I4JTv2lrPalqKgIt9tNu3btJKYRie9TE9moGDUgiUGGYUjNaE1NDR999BGjRo2S\npI3ExEQZcaPO4w+4mU/6YP6sBdSJFsmFmlC1otdoNBo7OZ4ldLzepX6//8fEPrWA0mg0Gk1EctIC\nSidJaDQajSYi0QJKo9FoNBGJFlAajUajiUgiKc1cZxFoNBqNRtAWlEaj0WgiEi2gNBqNRhORaAGl\n0Wg0mohECyiNRqPRRCRaQGk0Go0mItECSqPRaDQRiRZQGo1Go4lItIDSaDQaTUSiBZRGo9FoIhIt\noDQajUYTkWgBpdFoNJqIRAsojUaj0UQkWkBpNBqNJiLRAkqj0Wg0EYkWUBqNRqOJSLSA0mg0Gk1E\nogWURqPRaCISLaA0Go1GE5FoAaXRaDSaiEQLKI1Go9FEJFpAaTQajSYi0QJKo9FoNBGJFlAajUaj\niUj+H84bQOcjSV96AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f883c517cf8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_multiple_images(outputs_val.reshape(-1, 28, 28), n_rows, n_cols)\n",
    "save_fig(\"generated_digits_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "n_rows = 6\n",
    "n_cols = 10\n",
    "n_digits = n_rows * n_cols\n",
    "codings_rnd = np.random.normal(size=[n_digits, n_hidden3])\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_variational.ckpt\")\n",
    "    outputs_val = outputs.eval(feed_dict={codings: codings_rnd})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Interpolate digits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAocAAAB8CAYAAAAb3hoRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGH5JREFUeJzt3XmwVnUdx/GfmVaoQC6RGxpQQZZIbqAjqTgaq2lQYmPT\nBqP1h6M1ueC+zNA41eQyoI4QNRVEloojhWGFpIwSICoqCJgCrqQoWVZqf/H1/ft5z/UCd3mW9+uv\nD5d7nvs853fOec6c72/Z4e23306SJElSSim9r6vfgCRJkmqHN4eSJEkK3hxKkiQpeHMoSZKk4M2h\nJEmSgjeHkiRJCt4cSpIkKXhzKEmSpODNoSRJkoI3h5IkSQreHEqSJCm8vxP/los4d7wd2uE1bKeO\nZRvVPtuoPmxvO9lGHc82qn0ttpFPDiVJkhS8OZQkSVLw5lCSJEnBm0NJkiSFzhyQ0tTefvudfrVv\nvvlmi7+z4447Rt5hh/boE187+PmZ+Tkb7TNLklSPfHIoSZKk4M2hJEmSgjeHkiRJCvY5bEfsS/e/\n//0v+7833ngj8lNPPRV55cqVkUePHh15p5126oB32PG4D956663I/Pw777xzZPazlKRGVnV95M/f\n9753ntnYJ1tdxSeHkiRJCt4cSpIkKVhW3k4sB7z++uuRf/WrX2W/d88990R++OGHIw8aNCjyyJEj\nO+ItdjjuA2IZ5IMf/OB7/o7qD0tj//znPyO/8sorLf7OLrvsEpnlM/48pZTe//53Lk1VpTWPnbap\nmkbqX//6V+R//OMf2Ta77rprZLZFVbuU3UOqfq9RcXqy//znP5GffPLJ7PduvfXWyOxS9Pzzz0f+\n2Mc+FnmPPfaIvO+++2avtc8++0Q+8MADI/fs2TMyz6sPf/jD2fYf+MAHIjdDGxGvSf/973+z/+N5\n0a1btxa3b+t0bG3Zrzx2yvOI10jmzuCTQ0mSJAVvDiVJkhQsK28DPlLetGlT5HPOOSfybbfdlm3D\nknNrj/rrUVVZuUqzlTCaBY8Dlmp4jsyZMyfykiVLIg8bNix7rRNPPDFyjx49IleVWSw3V+PMCZwp\nYdasWZEfeuihbBvuw759+0bee++9I++5556R999//2z7gw46KHJV+9V7O1XNTrFs2bLI3/ve97Jt\neMyztMny8dq1a1v8eyxXp5SXP/lanA2CJeaxY8dm219wwQWR+Z1U7+1CVSuTrV+/PvIDDzyQbcM2\nGjp0aOSDDz44MrtNsBTMdkgpPy6eeeaZyH/9618jr1q1KvJuu+2WbX/hhRdG5nnUGW3kk0NJkiQF\nbw4lSZIULCu3ER8XP/3005EnTpwYmY+nOXKzxAmux48f3+LP6wlLRXx0T+Wk4FvwM3f2aCxtP5Y3\nPvShD0VmmeqJJ56IzFGZPCZ+97vfZa/L0bMnnXRSZJbf+Pd4HLVWYm6ksmZrWNZnmfP888+PvHTp\n0sgcuZpSSgMHDoy8efPmyA8++GDkdevWRX7ttdey7UeMGBH5u9/9bmSWzep9//M7gSXeadOmRea+\nTyk/5rt37x6Z3yOjRo2KzJJjWfrn/l+xYkVklkxZypw+fXq2/Sc+8YnIp59+euR6/R5qCY8xfjdx\n37388svZNr17947cv3//yB/5yEciV11Hyi5WPEZYFuaMJcuXL4/86quvZttPmDChxe07g9/GkiRJ\nCt4cSpIkKVhWrlCOOlq8eHFkjvp69tlnI/ORMkeMpZSPaOIoP06CXa9lFn5ujqjj43o+Lue+6NWr\nV2SWCVNq/XH9FizTsJTPbcuSGf/dSCWUrlA1ETJHSXJEPktZGzZsaPF3UkrppZdeisxzb+PGjZF5\nHg0ePLjF91GeU5xIuF7Ptyo8R1588cXIkydPjsyyMPcTy2cp5eu877fffpE5ypKlsXISbf7eCSec\nEPmII46IXJ6XW9RLu/A74rnnnovMBQ/KrjL8/D/96U8jc+JrbvPZz3428rhx47LXYpmU17758+dH\n/uEPfxiZJeqUUrryyisjc0T6UUcdVfn+603VdxOvHZ/73OeybTgin91jygmq24LbcF9y5DPfFycz\nT+nd34mdqb5bXpIkSe3Km0NJkiQFy8rAEmU5iTVHk3HdWOLEmGVZmeXLPn36RG6EMhfLKyyz33LL\nLZHXrFkTmeUkPsIvH+9zf3KE+OrVqyNzfVKWKZnL0jEnWD7zzDMjf/KTn6zcRu+tapQwSzvsasDz\nrTz2OYJv9uzZkXnusTTEUcyckPnb3/529rp77bVX5Kr1vusVz8Pbb7898qJFiyLvvvvuka+++urI\nHF2cUr5v2H6f+cxnIrMtWUpNKR8tzVG5xNetx/Ilj9lHHnkkMo9Rrn+cUt4liSXE1rpCbFHuo6rv\nm5NPPjky2+u0007Ltn/sscciX3XVVZE5c0BXljXbA/clS8Tc92WXpfZcz5iv/cYbb0SeO3duZHbJ\nOPLII7PtucZ5Z6u/M1KSJEkdxptDSZIkBW8OJUmSFJq+zyH7AVx00UWRr7322uz32IeG/dE44z/7\nM73wwgvZ9uy7cfzxx0fmVC71ivuQff3uvffeyFzMnPuS+499YFKq7h/G/oecnoGrAbz++uuRyz4l\n7P84a9asyN/61rciT5o0KXKjLkrfkTgVzc9//vPIbDuuUsC+qinlqwFw+poqAwYMiHzqqadG/vSn\nP539XiP3JeV5dccdd0Tm6h2cDoX938o+0sQpU7hKxMiRIyOX5xj7fXKalqqVbLh9vZxj3C/sp8fr\nYbdu3bJtDjvssMjtuVoPt2dbfvzjH488bNiwbBteO//2t79F3rRpU2Red+ulXarw/bO/Znsqp8Dj\nilALFiyIzH7AnN6r7CPdlSsK+eRQkiRJwZtDSZIkhaYsK/OxPx/v3nzzzZE5zUZKeVn40EMPjczp\nT1jWLBfQ5ooRY8aMiVyPZa6yhMRpGPi4nqVd7nOWY7gtp9lIKaVjjjkmMstULIGwLP/EE09EXrt2\nbeTXXnste12+L7bT1KlTW/wcF198ceSqspjy/fr9738/8l133RWZZTYeB+WUESwTcyqUoUOHRuZ5\nxBVWWivXbcsqB/Vi8+bNkVesWBGZq2cwb8s0HSxRs/zYr1+/7PdY2m/LKhNVKyDVMpYQ2UWC04/w\nul/+X2dcP9jGZ5xxRvZ/9913X+R///vfkdk9oZFwf3O/lKVgHotVmXjdu//++7P/40pF7JJxyimn\nRGbpvzyPuvJ65ZNDSZIkBW8OJUmSFJqmrMwyMUuOl112WWSWPsvVEziSdcKECZH/+Mc/RuZj+nL7\nY489NnK9r4pSvmeWWrmAO3H/85E+yyzlwvJDhgyJzFVVOOKPi5ZzlB7/BkfIppTSpZdeGnnevHmR\nWR7g6i6jR4+OPGjQoMj12CWgvbEENWXKlMgcLcuSFfcZz4kvfelL2eseffTRkbn6SdUow3o8j9ob\ny15cdYHnyN133x2Z7cLSb/lazDyXWMbmOZJSSgcccEDkrS2NtVZirqV25vscNWpU5PXr10dmKTGl\nfOR2Z3wW/g2uzJVSvnoKuxvwWtsMyrIy7wOee+65yPx+4DHNn5fdCA4//PDI7JrG7za+Vi19p/jk\nUJIkScGbQ0mSJIWGLitzNCRLIEuXLo381FNPReZISk4Qm1JK5513XmSWDebMmROZE1+Xk++yLN3a\nhLP1qKo8wv1U5aMf/WhkLoaeUl6aZzmRj97bMkF1WU7hI/358+dH5vHyyiuvRL7xxhsjX3PNNZE5\nQra1v99IuI9SSmn16tWROdqbZSqWU8aPHx/5iiuuiFwuML+9C943I06YWzWqnpOTs/TPkd8p5d1A\nHnroocjsRsNRliz9p5SXytpyXtT7ucNR3DR8+PDs37ymVZXP23Nf8G/wnEwpv3ZyEnp2Sdhzzz07\n5H3VkvKatnjx4siXX355ZJaY99lnn8iXXHJJZM5kklJeoq/af7U6g4JXYEmSJAVvDiVJkhQaqqxc\njjpiKZnl31//+teRWb7imsdlOYClFZYZuWYwyzrl6Euun9hoJbOqdT1ZdmXJnb/PSUK5L1PKR/qx\nNMn915a1WstJXTnikiUFbsMyNv9eo3UJ2Fo8p1JKafLkyZFZsmR7cSQnZwfoynVDGxGPzW9+85uR\nqyZ3//GPfxx5xowZ2WuxhMb1YdlmnKT+yCOPzLbnuuiNitcOTn7MEjvLsinl3VU4srXq+OfPy/Jj\nW75H+B45CjelvLsNy91cNKC1ieobBUfzp5TS3LlzIz/yyCORucb7yy+/HJn3E1ygIaW829Tee+8d\nmZP88ziopRJzY7a2JEmStok3h5IkSQreHEqSJCnUfZ/DqpVPUkrpl7/8ZeRly5ZF5lD97t27R2Z/\nqunTp2evxb4HGzZsiMz+CpwKZ8SIEdn2zdJXjdMjfPWrX4185ZVXRubKDM8880zkH/zgB9lrsW/g\nfvvtF3nw4MGR2aeD2P/0sccey/6P7cf+iOzfw/4h7L/VbKsHpJTvy3K1GfYl5e9xX3LaB54H9jNs\nX9yfZ599dmS22aJFiyIvX748ctlfm/9mP0X2QVu1alVk9h1OKZ/mppFUrULDzHPigQceyLbv0aNH\n5IEDB0bmPuZ3WtU5Vb6XqpU12F6ctq18Pa5WxL/P7Wtp9Y7txc9ermZ28sknR+Y5snLlyha3ue22\n2yKX5wG/X7gNV07himvlKmHle+tMPjmUJElS8OZQkiRJoebKylWzxvPnXOj61ltvjTxt2rRsG5Yv\n+die5V9O08DpG0osH/OxO98XS6qc8iGl5imhcT9PnDgx8syZMyOzRM+yRflI/s4774zMx+v33Xdf\ni9tXlYjL6SRWrFjR4ntnybN///6RWbpme5fHajO08e677579m/uWbc+24NQQ5557bmROd9MM+64z\ncaqNv//975F5jeN1rMQSIrvecHop/g6vqSk1R3tyejNeUzgdSVnK5bWE1xvuS3ZdYVmZ32cp5aVs\nrn7C7x6+1po1a7Lt+d44zc1xxx0Xuep610jtW5bLjzjiiMi/+MUvIrO7wMKFCyP/9re/jVxOZcP7\nBu6zRx99NDKn9/rUpz6VbT9o0KDInT2VkE8OJUmSFLw5lCRJUqi5sjIfo7M0xRGn1113XeS77ror\nclkmYZn3oIMOirx69erIfBzPxdO5ukdKeVmTo5r5fll+aaSRXduK++M73/lO5Isuuigy93nZfiyH\nsT3YfpzNn4/dWbIsS/xcZJ44U/2QIUNa/DlLA41UWmkNP2evXr2y/zvvvPMi81y6++67I7O7AEf2\nnXXWWZFraWWAesX9z33LUhfPMe5zzgaQUkpf/OIXI7PkyNkF2KWgvF42w7nBMi2vY9yv5X7Zd999\nI7PkW7W/eE3j91m5Pdu+qqsNu2OV75krvHB0O48jrvLVSO1bfhZ2j2H7caT5AQccEJkzk5TXMf6b\nK0hxZam//OUvkRcsWJBtf8ghh7z3B+ggPjmUJElS8OZQkiRJoebKylyYnKNaJ02aFHnJkiWR+Qi9\nnGiaj3Q50oulFZZMOBKTEy2nlNLjjz8eed26dZE5gqtq4tiUGusxfFtx/5922mmROdp43rx5kcvR\neGxblo85AoxlF7YlSyY8plLKyyt8jxyJOXz48MjsUsB2bJbRyvxcZdmEkx1ffPHFkdlGLOOzFMf2\nLktmem/lxNXsusEyYdV1aezYsZFZ5kopvxbec889ka+55prI/fr1i8xJ41NqjnNhwIABkXle8JpU\nTpq/tSNOW9uPbEt2Y9q4cWOLv7PHHntk2/N7jNdXdufhaFl2Pdh1113f8703GrYdvxN47Lc2UTlL\n1GPGjIlcjnCuFT45lCRJUvDmUJIkSaHLy8plaY4jUWfMmBGZZV2OwOL2ZVmS5RS+LktYnPCSE5Sy\nxJhSPuFp1eSxXEPW0cr5I3bu8wsvvDAyuw5wJGRKeUmGo/yq/gYn+OUkwBxdntK7y3FbcIQz3y9L\nM62VhRp1klhqbWQfR/ANGzYsMs89lqM4iW8zTqK8vThCNaV85gYe4zxmOVMAR5qX3QU2bdoU+Q9/\n+ENkTgTMbjxlN5pmwNG7LLlyYYVywn1ei/gd0ZZyc/ldyW40bC++LtuV18SU8q4fHKHM7jz8LM1y\nTlYtxLEtn79qdgv+DbYdc1fzyaEkSZKCN4eSJEkKNVcL4ON1Tg7J0ad8PFtVIkwpf7zOx/59+vSJ\nzIlcWVbesGFD9lrlv7dgOYWv29nrINY67o8DDzww8vXXXx/597//fbYNRxmzywBLkxyBx4mXOSKZ\n5ZOUqtdgZleCssy5BcsBZRvXS9mlam1wlqDaUg4pX4tlLpawHnzwwci9e/eOXNU9pPybahm7ZKSU\n0quvvhqZbXnYYYdFZimZ166yRP2Tn/wkMicu5znyhS98ocXXahbcF5w0/IYbbojMkd4ppTRu3LjI\nEyZMiDxw4MDI5cjvLcrrzapVqyIvXrw48tChQyNXzbSQUj6LANuff+fggw+O3EhtXLVmdKk9r0P8\nHvrzn/8cmd9z5UTlXXkd9A5GkiRJwZtDSZIkBW8OJUmSFLq8E0FZ72e/Pc74/+ijj0b+05/+FPnF\nF1+MvNtuu1W+1qmnnhqZ/QzZT4d92VauXJm9VtV0JuzTwQW47TNVjX1B2R+K/W5SyqfNYDux/ykz\nZ/lnGy1dujR7XfaT69mzZ+STTjqpxe2pEdp10aJFkTlVxbHHHhuZU/lwfy1cuDB7rVtuuSUy+8Bx\nmhq+1hlnnBG5V69ekRthv3YGXi/ZxzOl/LzitEznnHNOZO5n9jO74oorstf6zW9+E5l9Q7/yla9E\nPuSQQ1p83WbB4/rSSy+NzH5jU6dOzbZhXzOeh2wvrk7Dvs/ldF48Fjj9zPLlyyPzO+2ll17Ktmc/\nfF47+/btG5n9HxupzyH3V9knnefR9n5mjolg28+cOTMyv2u+9rWvZdt35dgFnxxKkiQpeHMoSZKk\n0OXPictyBB+Dn3XWWZH5eJaZj4dLLJuwNMLHxlUrCZSrrXCVAJYAjj766MiHHnpo5XtRy6oWrE8p\nX3GG00awnMKyMkvEZamAuM3w4cMjn3766ZF5HPJ9VU3xUgtaW6GF5wlXopk0aVLk7t27R+Yxzi4W\n5SoL/JvchiWriRMnRmbZhCWzWtuXtYr7qZzyhPufx+z9998fmSswzJ49O3LZ9YLHy5AhQyKfeeaZ\nkbmCUbNjiflHP/pR5M9//vPZ733961+PzK4X/L7h1CZsb3atSilfoYaWLFkSmdex/fffP/s9tjG/\nxwYPHhyZXT/qfXo2do/hND4bN27Mfo9TrbGrWtX3AK+B5ffOTTfdFPmyyy5rcfuf/exnkfv379/q\nZ+hM9d3akiRJalfeHEqSJCns0Nrs4O2s0/5Qi3+8DZ+TJeZyxYAFCxZE5uN9liUHDBgQmaXrTtQe\ntbkubSdqyyz2bDOWDTj6bv369dk2LL/ttddekVlWZbm7tXIttbE02iltVO4vlq2efPLJyFylgSsu\ncMQlR8WWpSWW8lky5kocHH3J/VrD6uI8Kq9RXKXj3nvvjczzgqVIlsDKEuXxxx8fmWWvHj16bMc7\nbnfb204d3kblecguLXPmzInMUf8cxczfL7tQsazPWTOYOTMHy8Up5d9Xxx13XGSOVuZrVZWx30OX\ntlHV6P6HH344cjkDQ1WXCs5+wnOPK3PNmDEje63HH388MrtJTZkyJfIxxxwTuYtK9y22kU8OJUmS\nFLw5lCRJUmiasvLWKvdL1aSZnCSTpeQuGn1ZF+WwjlJ1LJflGP57a9uMo97LEkAbJ0ztkjbivuGI\n1aeffjryvHnzInM0H0vHLDmllE/83paRfXWiLs4jdqlIKR9VzsnG16xZE5nHPkvEHLWeUj6qtou6\nyLRFzZeVW/3jOCd5Xdm8eXNkdmG68847s+3ZDYbYjYNlUc4gUG7P85XHVVXbb8U5XTNlZX5vr127\nNvLNN9+cbcOFNXitZLusW7cuMs8pdlNKKaUvf/nLkb/xjW9E5rWyBq6PlpUlSZLUOm8OJUmSFCwr\nN5a6KId1NZZNtnZSa5YQyrJyLY1WftcGbRxxrZRSnZ5HbGOONufISh6znCC5nNC6To6Rui4rN4ma\naSNe9zkKvJwEm2tTc6J5lqU5CpyZM2GklI/wruFJxC0rS5IkqXXeHEqSJClYVm4sdVkO62zbU2Jt\n7Xyp5bKytoptVB9qpmSpSrZR7bOsLEmSpNZ5cyhJkqTgzaEkSZJCm5Z0kBrV1vY/rJMpPiRJ2mY+\nOZQkSVLw5lCSJEnBsrKajqVhSZKq+eRQkiRJwZtDSZIkhc5cIUWSJEk1zieHkiRJCt4cSpIkKXhz\nKEmSpODNoSRJkoI3h5IkSQreHEqSJCl4cyhJkqTgzaEkSZKCN4eSJEkK3hxKkiQpeHMoSZKk4M2h\nJEmSgjeHkiRJCt4cSpIkKXhzKEmSpODNoSRJkoI3h5IkSQreHEqSJCl4cyhJkqTgzaEkSZKCN4eS\nJEkK3hxKkiQpeHMoSZKk8H+Dw24sh8ZUaAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f88386ae668>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAocAAAB8CAYAAAAb3hoRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGk5JREFUeJztnWmsXVX5h1+cUSlSsAKtbQGhgFDKDCpSS4tKtSjIUAiG\nxiFiIiYmTvjBOESCUVGw1BIEojjhQJjtICI0LQhSBAULnaBFkNGiUBzrh7++Pmv9zz6e2/bec87t\n83z6tffsffZZa++1V9bvfd+11YYNG0JEREREJCLied2+ABERERHpHZwcioiIiEji5FBEREREEieH\nIiIiIpI4ORQRERGRxMmhiIiIiCRODkVEREQkcXIoIiIiIomTQxERERFJnByKiIiISOLkUERERESS\nFwzhd7mJ8+Cz1WY4h/00uNhHvY991B9saj/ZR4OPfdT7tOwjVw5FREREJHFyKCIiIiKJk0MRERER\nSZwcioiIiEgylAkpPc+GDf+Nfd1qq02Lo/3nP/9Z/Jvna/qeTf3OXuMf//hHy/9fv3596pe85CWp\nn//85xefG27tISIi0g+4cigiIiIiiZNDEREREUmcHIqIiIhIYswh6DTGjTGDhDF2dczh857333n4\nC14wfJudv5vt+eSTT6Z++umnU++8886p65hDEdk4msaoLSXeuVdp6pdOsI+Ghs2ZezBQ6nlDE0Px\n7LpyKCIiIiKJk0MRERERSYavvzmIcBmXy8C0S3/zm98Ux7zmNa9JvcMOO6R+4Qtf2PK8/Qrtc7bN\nNttsk5q/WSu5v6EFw7CKv/3tb6n/8pe/pH7uuedSP/vss6m33nrr4rwjRoxI/aIXvSg1QzKGw/PS\nCZ1YxBERf/3rX1P//ve/T7106dLUy5cvT80+qtty6tSpqbfbbrvUf//731O/7GUvS81nOiJi++23\nT83+66TP6t/VD/3MtuRz8PDDDxefu+OOO1Lfc889qVesWJGaYyL7tA5HYr+MGTMm9X777Zd67Nix\nqXfcccfiePYfx+1+pym0ibBdn3nmmeJvvF/ZRpt6H/K+5jU++OCDqa+//vriGN4X73jHO1IfdNBB\nqVkObnM+K8PnjhARERGRTcbJoYiIiIgk2sobAZeHaY19/OMfT71gwYLimCOPPDL17NmzU9d2TD/C\nJfJOsq3a7Yoi/QvtNNrHTzzxROorrrgi9TXXXJOaoRYRESeeeGLqadOmpWZ4QlPWfztrpZuZiPX3\nd3oNTZ+pn7U1a9ak/uIXv5h60aJFqdlHo0ePTs2qARFldQFaa7TAHnjggdS0kSMiZs6cmXr69Omp\nadk1WZn9YCNHlH3J0Imrr7469YUXXlgcc+edd6bm2EebkPYvx8p169YV52LoUv23Vt9BWzIi4lOf\n+lTqzWmfDgWdZn3zfuf9xvd2vZPXUFQToa09d+7cljqifC5PPvnk1AMN1dgYXDkUERERkcTJoYiI\niIgkPW0rD7Rg6FAth9NCOP3001NfddVVqWvL56c//WnqtWvXpp4wYcIgXGFvwP5jNt+LX/zi1Juz\nmGen90s/2Cb9Au9ztj9tE1qU/AwzlO+7777ivNddd11q2mz77rtv6pe//OWpaQXxO5hdW0NrZqjY\nnPc4x6GIiIsvvjg17Xvyile8IjXbn9muERFTpkxJ/dRTT6X+7W9/m/ruu+9uvJZly5al3nXXXVMz\nk7bfYV88+uijqb/xjW+k/t3vflccw7Hv2GOPTf3Zz3429bbbbpua4+Zjjz1WnItWMrPQL7/88tQ3\n3nhj6u9973vF8QceeGBqWs79sElDu+eo6Z3CsYCa1n3E4FUQ4bnYdwxD+OMf/1gcw3CNcePGpR6K\n7HJXDkVEREQkcXIoIiIiIknvrx//m27vCUqb7JOf/GRqLgm3KyrL41nocrfddkvdr5nLTcWPaSeS\nkSNHpubyeN1mAy2Yy+9me9cWP9u5qYCo1vPAYZsx+3GnnXZK/YY3vCE1reS6vf/85z+nvvXWW1Ov\nXr069e677556/PjxqVnUtrbiJk2alLobtvKmwvudbRFRjiu0eV/1qlelnjFjRupTTz019ahRo4pz\nvfSlL03NzE5mK7OgM7PTI8rQmcsuuyz1XnvtlZoWK2mX0d1LzyXHGxYdZxY3LeKIsqLFaaedlrrJ\nymQbMYwiomwnhltMnjw59Wc+85nU8+fPL47/9re/nfqwww5LzSz2XmrvTYX9xXG/Hgf4mzdndQO+\nh1hBgIXS680AGIYx1BnlrhyKiIiISOLkUERERESSnrOVB7qMuzmXfXmuer9F2jE33XRT6rqAZhO0\nOVm8lPZPv9rKbHfaeEuWLEnNLCwWfOXen7Vt0rSPLjPN7r///tTf/e53U998880trymibGfuef32\nt7899SmnnJKaS/2d3mPdLrY8WLTLCGfBXYYL0FqjFUrrmNZlRMQjjzySmnvQsu8mTpyYmtYQLVH2\naURpLfUjHG9oX0aU9hQtrIMPPjj1WWedlZp787aDhcff+973tryWOXPmFMdwXKMd1klFgToMpFcL\n5fMeX79+fWpmhLPtIyLe/OY3p+4kK3ZjsnLZr7Sx6zCfW265JfWVV16Z+owzzujo+/sNWsm8d+vf\nyPtvU+89nmvx4sWp2S9N905Eef8MdRiMK4ciIiIikjg5FBEREZHEyaGIiIiIJD0Xc9iURt4EP1N/\nvul4xhtQ33bbbam5yXVEGQPF8zLuhDFy7cqnsOp50/X3U6wHr5vxZffee29qxh/ecMMNqffee+/U\n9cbwLK/BOENW+v/mN7+ZmjFYjC1rdx9xBwOWF/jlL3+Z+n3ve1/L6613EmBMCO+rfthxYHPQtHvH\nr3/969QXXXRR6ttvvz01d+GIKONPGffDfmUcK0tCnXjiialZOiXi/5eK6Dc4rqxatar4G9uc4xLv\nWT6f7eKpmsY4Hv+BD3wgNcfHiLJMEXdb6SSGq35e+Jt7dVxcuHBh6scffzw1Y80jBm9nC7YL24+7\nC+25557FMYw5ZOz2QHcm62U25n7ZlN9f9zefEb5fOHbtvPPOqd/5zncWx8+aNSv1UL9HXDkUERER\nkcTJoYiIiIgkPe13NVnM1LQb6zIV/DeXdH/84x+nZvkHlj9henl9LSNGjEjNXRpoWXGD+vp4lnZo\n2qGjn+ByN61VWihsZ9qM3Flg5cqVxXknTJiQmjbVvHnzUtOO5L3QbleFplAAwnuBO0zQCuX3RURM\nnTo1Ne032gb9Th0u8fTTT6fmPc/dOtiWtP5ZLqou48QSQ2PGjEl99NFHpz700ENTc1eHdvZLv1v8\nbKdXv/rVxd/YN7SYWXrrIx/5SOp2bdE0FvHZ4dh1yCGHFJ9jyRaW4+ikfEs/wt/CcJg//OEPxec4\nXtHyHay2oI2/xx57FH874ogjUjOMajj1C2my3usxjeM6beKm3bx+9atfpX7/+99fnIv9P23atNR8\nJ7D01nve857i+B122KHldw4FrhyKiIiISOLkUERERESSvvFYmixmbvi+YsWK4hjumHH55Zen5tI+\nz0Ubus5qPOaYY1J/7nOfa/n9X/3qV1MvX7688fpHjhzZ8v/7Ff4G2sps57Vr16Zmm9Fi5mciyqw/\n2lG0R7gxPavLM3t13LhxxXkfeuihlpoZe0888UTqO++8s+U17rLLLsV5aeFwV4l+h89IHW5x8cUX\np/7+97+fmn3PtmCm/pve9KbUdVvSlmTGMS0Y2jy9uovG5obPGkNaIsr2YLb8rbfemvonP/lJ6hNO\nOCF1vftCJ+MS7TeGAdT/HujODp1Wneg2bG/er9wBi2EXEaWF2bS7VruQmE2htpW5QxHHyOHwThoI\n9f3F9w7nEAxtYt/zHcbxKaKszkBbmWEf3LmGoQb19ww1rhyKiIiISOLkUERERESSvrGVCZfmuTR+\nzTXXFJ9jVjJtQi7VckmZWcQspBsR8eUvf7nl8czKnD9/furaTuCyPW3lbi4bDwa0mrh0zuV2ZoBR\ns48iIu66667Uhx9+eOoZM2akftvb3paaGWC0smordOnSpS01wwp47TzvgQcemLouWEpbu86+HS7U\nbUnLnWEdvK/Zd8xAf9e73pV67NixxXmZCTvcnpFNgZZf3Wa8NxcvXpya4+Xs2bNTv/KVr0zNzNWI\nsv15PENCGJLBcbg+ZqCWf21r9kPIwHbbbZeambAHHXRQ8Tney2wzHsPPcBypnwO2S5MVzH5Ys2ZN\n8TeO1QzP2RJoZ50zdOjKK69MTVuZ7wqGvZx//vnFufbff/+W3/GnP/0pdVOFj/91nYONo66IiIiI\nJE4ORURERCTpe1v52WefTc1ilBHl0i3h0i2zkidOnJj61FNPLY5hZhntTmYu0xatC8yymCUL+Q63\nzDDaE5///OdTc+n961//empmhtGyiij3n+Rex7Rv991339QsVk1bmHZARGkl82/r1q1L/frXvz71\n5MmTUzPjjNZbxPC1P3mP1lnYDAtg/zFcg5n7xx57bOrx48enrm2t4fZcDAb1/XfGGWekph3J9qdl\n9olPfCJ1XcSadiiLm/N5ZR+zUHlEma1My7UfLOJO4T3K8YZhFPWYsGDBgtQcu1avXp2adj3ty7ro\n+fTp01Pz/cLv5DPJDQciyrACWpvs1y3hOayzlblPON9BDANgG3MfeGYuR5TPKPuVxbG55zX7pL42\ni2CLiIiISNdwcigiIiIiiZNDEREREUn6JuaQ3jtT8Jlq/uCDDxbHMO5v2223Td1UpoHlZy688MLi\nXIw9WLJkSWrGqfFcdQzV1KlTUzNmbrjBuAjGGs2aNSs142POPvvs1PUON4y3efjhh1P/4Ac/SL1w\n4cLUjHNiWaK6rBD7ZvTo0am5Y8db3vKW1CxDxHunjgHZEuJz6ljaAw44IPVVV12VmrFOd999d2rG\n7XRSikM6h2UzDj300NRs2zvuuCP1smXLWuqIiEsuuaTldzDWiuWp6ng4xubutNNOLY/v9z7n9TN+\nlu8BvjciIkaMGJGa7zTG/N17772pGe/Jd01ExLnnnpua5b3e+ta3pn7sscdSL1q0qDiepY+a+mJL\niD+sfxdjCBljzb7jOMi49Y9+9KPFuRiLShgTzHfNnDlzis/xXWXMoYiIiIh0DSeHIiIiIpL0nK3M\nZWxqlszgriS0suoyCSwVwGV72iHclJ7L+SxXE1GWsqGmhcBlX5bIqa9luC7P1zRZUCzBwM9ccMEF\nxfG0VGgNP/PMM6lZruiBBx5oed5Ro0YV52WZmilTpqSeNGlSatpEdcmQ/1CXQOC9MFzL2tS/mTYx\nwz0IrRXuqHLSSSelHq47ygwmDLuIKMto3HTTTamfeuqp1Bx7aFnxmYpovpc5xnJMru3TH/7wh6kP\nO+yw1LTj+t2y5PWvWrUqNUuhcHeliIjjjjsuNcNreP8zBInPy3nnnVec6/7770/NsAC+e/h8rly5\nsjie/UdbmmE0dRjJcKS+93bffffUtN4ZOsEQNpaHqm3kpnkDw25+9rOfpWa5m4juhqANzzeYiIiI\niGwUTg5FREREJOn6mnFtRXGJl9bG1VdfnfrMM89MzQyuegmWy/bcPYNZW007dNT2WVNmJTWX4Pnd\nERH77LPP/zx+uME2ZF/Szthll11Sz5w5szh+/vz5qbn7DTPwmvqs7j/y2te+NvWECRNSjx07NjXD\nEJqgTRBR9uVwspXZlvUuNrRXWBGAOxfxuWBGOp/dOrt/OLXf5oR9cc455xR/Y6YjbWJmXDLEgrsx\n1OMwj+dYRpuS/ffkk08Wx7O6AK2y3XbbLXW/W8wc0+bNm5eaY9WOO+5YHEObmfYvfzMrLXA3rTe+\n8Y3FubjLBnfnYggOP3PbbbcVx3PHlLlz56bm+5XXO5x2t2Hf1WMa70W+k3gf8x3GkII6PIPnWrx4\ncWreI+x7hkZFlO+qocYRWEREREQSJ4ciIiIiknTdVq6Xqpsyer7zne+kpq3I5WFaxBHlMjotL1oe\nXB5uyuSLKG3Geun4P9AmOeqoo4q/cXl+S7HMuKTObEZmUtL2qDO9Jk6cmHrMmDGpr7322tTsS1pj\nbGMW444oM6ebsm2bLGpmiNLKiSjvZYYx9ItNRpqs5DrjkZl9H/zgB1t+jlYyM/747PE5jthyQi8G\nCu/3r33ta8XfaO1yLNp7771Tf+UrX0lNi7duf45/HFdZgJ6ZtLUdRmtz+fLlqdn/fEap62shvTR2\n8jpZ6YL3e53F3VQEvOke5+frChgMg6F9ze/k//M+iCjDdljImRU8jjnmmJbf34/PZNOGF3ymIspx\niW3MiiNsi3b3JOczrIDCduWzuuuuuxbHd7Ode+dJExEREZGu4+RQRERERJKu28r1simXYWkf0zLp\nxPKLKIu/0grmMjAz9ljsuC5eyqVnFl5u2h/zyCOPLI7vJPt1uMGM1S984Qupr7vuutTsi0MOOaQ4\nnnsdc0l/9erVqWkF8/t4H9V2NfdjZgYaj2EWNbM1eY/Rko4o7zHeV/2S5ddku9xzzz2p169fXxxz\n/PHHp2boBm2qn//856n32GOP1Oz7dtnlg0W/ZMjyOrnnO8e3Gt6z3/rWt1LTSm6qDBFRWmtN1iaf\nnfq+YJblQPdW7uW+IBx7+H5iW9T7xTcVim+iXVvwvuAYw6odLK7NPYPr4x999NHUHF8ZUtJuX/le\nhb+R7wFmC3O/8YiyaDurWfAd3ml4A/ulDlVrdd7tt9+++Ju2soiIiIj0BE4ORURERCTpuq1c2xnM\n3KF9y6wr2ld1Nhjhci2LvzZlwXJJlxZlRGnNNO2nzKVmnrf+W78syW8qXNJnxiIzlNkWtf1L25Y2\nQFOGMm1h/j8tsohyz+amfWdPPvnklueiZcNMtvr4foG/jQWtmenPZ6cuysr24HNBy5FZrcwUH6jF\ntqXCdl2wYEHqdoX6P/zhD6dmiERTVnBtC99yyy2pr7jiitQs+MvncMSIEcXxRxxxRGpmyXJM7tRW\n7kbIQSewLZsKhd94443FMUuXLk39ute9LnVT6Em7wv4cL/m+YhFthsHUfcxjeM0sjt1Jf/UCTfcI\nfzMz6i+99NLU9WYGU6ZMSU1bfmMy5Xnu2bNnp24a++rQOG1lEREREekJnByKiIiISOLkUERERESS\nrscc1jB2YK+99kp93nnnpWYcx49+9KPUdbkYxkextAxjbfh9jJ+6/fbbi3MxHpFxjowJ4K4YddmA\nXo7XGCwYr7HffvulZvwgY0LqNv/FL36Rumk3DcZuMF6VMVCMUY0o49722Wef1NOnT0/N/uPvoK77\nlP/u5fI1bDPGQLFMyjbbbJP6pJNOajwX45MuuOCC1NzRiO3COCvuXMO+q4/Z0mGsE8ts1CU4eG8y\nlpbxZPwMd/XgzikREcuWLUvNnYBYjoOlTaZNm1YcP3PmzJbHNO0KwnG43W45vQR/19lnn52a49ua\nNWuKY84666zUX/rSl1IzLpTPAktKrVu3rjgX49MYf8j2YowqY60jyvcrf8vUqVNbXku3aRrrI8rY\nPv5+vl+WLFmSmm1Rx/mxJFlTzGfTPVnHLzKOe968eamZTzF+/PjU9W5e3cSVQxERERFJnByKiIiI\nSNI7a8b/hnYcl45p2dLmoq7T2Zvsx05K0dBWiygtTtrKtB95LfUG2lsitLBmzZqVmiUFaCXXtgkt\nAfYNSzWwhMQBBxyQev/9909NKy6iLEHD/muqgE/bgfdLbUfwerlDSq/BZ+Gcc85JvWjRotRsC9pk\n9TNG+5EWGO0V7jLw7ne/OzWt/6GyDvtlVxTC6/zYxz6WmuVmIiLuuuuu1JdddlnqG264ITX7qKkk\nVP2dfI4ZhnH66aenPuWUU4rjaTl3UgKkXV/0aj/xujimsPQP7eaI8j1y3HHHpWYb8dnju4rhMBHl\nO4qhA03lZ0aPHl0cz51r9txzz9T17mC9AucD9TjUVCqObcb24nNQl8M799xzU3McHzduXMtr4XuA\n77aIiA996EOp165dm5rvrTlz5rS83m7jyqGIiIiIJE4ORURERCTZagirz/dMmftOfnNtGT7++OOp\nr7/++tRcHmYm5siRI4vjhyh7dXP4L4PST7RHmEm5cOHC1Ndee21xDDeDpyUwY8aM1AcffHBq7krD\n7Lu67ZsyJjuBv4M7vUSUoQ9trLSu9BHveWZ+f/rTn0590UUXpX7uuedS1xl4TdASOeqoo1JzZwA+\nL92wC/lb2jyTPfscsR/r7P7jjz8+9SOPPJK6qToDdd0WtCBPOOGE1GeeeWZq2pLtnrGBUmcrD+Kz\nNOh9RLs3ImLu3Lmpzz///NQc6/jskbodWIWBzx4zXvkcHn744cXx3CmMfbn11lunZr9SD6B/u9pH\nvJeY0c1+4DsooqzAwDGd4THcXWbFihWpV65cWZyLljUrnvD7J0+enLpLVS5a9pErhyIiIiKSODkU\nERERkWSLtJU3hqblabZfU7HkIaRn7bAm2K61ld9ke9FeacpSGyzaFevt8Fq63ke8f2nxX3LJJalp\ntTCTtd0m9cxIpy3ZrnB4j9L1PuqEui/uu+++1AwRWL16dWqGx/CZov0YEXHaaaelHjVqVOqmgtZd\noidt5XZwzKDlzILkN998c+pVq1alru3mSZMmpWYmLYtb8zmsN4nYFAuzX2zl4kQNoTUPPfRQ8Tlm\nFTOjnwWtuXkAi2azAkNEGfbE8Aw+Uz2wYYK2soiIiIi0x8mhiIiIiCTaysOLvrDDhiu0jHotW7mj\nk3YwFvSAlTgU9GwfSUHPWJbSiH3U+2gri4iIiEh7nByKiIiISNJzeyuL9Cud7CHby2whlrGIiPwP\n+vttJiIiIiKbFSeHIiIiIpI4ORQRERGRxMmhiIiIiCRODkVEREQkcXIoIiIiIomTQxERERFJnByK\niIiISOLkUERERESSrTZscF9rEREREfk/XDkUERERkcTJoYiIiIgkTg5FREREJHFyKCIiIiKJk0MR\nERERSZwcioiIiEji5FBEREREEieHIiIiIpI4ORQRERGRxMmhiIiIiCRODkVEREQkcXIoIiIiIomT\nQxERERFJnByKiIiISOLkUEREREQSJ4ciIiIikjg5FBEREZHEyaGIiIiIJE4ORURERCRxcigiIiIi\niZNDEREREUmcHIqIiIhI4uRQRERERJJ/AW056Ey8bUMzAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f88333e6550>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAocAAAB8CAYAAAAb3hoRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGahJREFUeJzt3WuwVWUZwPHH0i6kFalhGCkKQiqaZgZmamoTTje11Gkq\nI9MpvmTaTE3XaSYL0SntMmXlrSRRx8QyHETHDFOKINQE8ZKIQN7yFlZmF/vk0/99O+u0gXP2Wfuc\n/+/TA+y192a9a629Zj3P+7xbPPvssyFJkiRFRDxvqL+AJEmS2sObQ0mSJCVvDiVJkpS8OZQkSVLy\n5lCSJEnJm0NJkiQlbw4lSZKUvDmUJElS8uZQkiRJyZtDSZIkJW8OJUmSlLbs4me5iPPg22IA3sNx\nGlyOUfs5Rr1hc8fJMRp8jlH79TlGPjmUJElS8uZQkiRJyZtDSZIkJW8OJUmSlLo5IUWSJEkdePbZ\n/87H+fvf/57xlltu2Wc8kHxyKEmSpOTNoSRJkpI3h5IkSUo9U3P4z3/+M+PnPa/ve9qmv1e7sI5i\niy0Got+wpH//+98Z8xxrwuul56E0NHiuPvPMM8W/8c+PP/54xi95yUsyfsUrXpHxQJ7H3k1JkiQp\neXMoSZKk1DNp5ec///kZN6VM6r83VdIeHJumeHPfl+PdFGtgNY0l2y488cQTff79k08+mTHLRiIi\nxowZk/FLX/rSjJlO2ZS0KFOvvVKGwn3DffznP/+5eN2SJUsyXrx4ccZr167N+EUvelHGTEe9+MUv\nLt5r0qRJfW7DePz48RlzvCKax2m4+sc//pExj/F77rmneN2DDz7Y5/Z33313xuvWrcuYxzX3fUTE\nqFGjMh43blzGY8eOzZjnzg477FBsz/F/4Qtf2Of30uDgefzUU09lfN555xWvmz9/fsZMK3/mM5/J\n+KijjsrYtLIkSZIGhTeHkiRJSq1OKzOd8q9//SvjrbbaKuOhTh92mhY1tflf3Gedppw6SUvzGGk6\nXiIGr6P8SMT9/7e//S3j1atXZ7xs2bKMb7zxxoxXrVqVcX0cvO51r8v4Pe95T8avf/3rM2ZajXiu\n1edn07HT5vOTJTV/+ctfMr7yyiuL15122mkZM2XPlDFTiTvuuGPGdVpxwYIFfb4XP5/e+ta3Fn/+\nwhe+kDHTnMMpxcwShaeffjrjb3/72xn/4he/KLbhMc9xOfzwwzOeMGFCxq985SszZro6IuKvf/1r\nxrfcckvG8+bNy/ihhx7KmOdORDlGL3jBCzJu87kwUIb63Ofv0ze+8Y2MZ8+eXbyO90A8X5vugQbS\n8DlTJUmStNm8OZQkSVJqdX6tkxREtx4P83P4qPexxx7r8zWjR48uth9pj+370zTLtL8UfdM+a0rt\ncGZX/b58PM8U80gfl05wf0eUM2aZ2rr88ssz3rBhQ8ZMpzCV+cADDxTv+9vf/jZjjt/LXvayjCdO\nnJgxZ3L2d0zx+w91SUqn+J05+/Wcc84pXvfwww9nzP/31KlTM/7Yxz6WMY/9NWvWFO/FVPK9996b\nMWdE87vMnTu32J5jzu/J2bO9jsfM+vXrM77iiisy5jUpopwFfsIJJ2T8zne+M+N6VvJz6nOPY8zZ\n0tdcc03G3/zmNzO++eabi+3vvPPOjPfff/+M23wubI7N7YwxkJ9/6623Zvyd73wnY5YKRJTn6IEH\nHpgxyxBMK0uSJGnQeXMoSZKk1Lq0ctPsU6Zy+RiVM/nqx6ub87i1fgTNmWKcDXbppZf2+ZoZM2YU\n2zNtwFlqw/URfm0g/59NaUM2E/35z3/euP2UKVMy5qxYHkucDdZp6nkkjGWdJvvlL3+ZMdNZ3Bdv\neMMbMmbK7KabbsqYacyIMkV63333ZXzJJZdkfNBBB2U8efLkjNmAuU7R8d/aPF5Ns6qvvfbajJnK\njCivK29/+9szPvPMMzN++ctfnjHLO3itjSivtxxzNtc+/fTTM7799tuL7fk92dj35JNP7vPzexFT\nuUy9c1++8Y1vLLZh8+Kdd945Y157NgXLlqZPn54xywBWrlxZbHPXXXdlvO+++/b5Xr2uKZXcrXOf\nn89m9B/4wAcy/tOf/pRxve/ZXeDII49sfN1g6O2zU5IkSQPKm0NJkiSlVqeViamNRx99tM/X1+t7\nMqXUyWN7vhdnu0ZEfPGLX8x4zpw5Gdezi56zaNGi4s9HHHFExkyzcPZem9Ncg2VTZpBxGzbl/d73\nvpcx93E9RtyeKeOtt946Yzb1PeSQQzJ+1atelfGee+5ZvC+3r9eq7WWcJblixYri366//vqMd9pp\np4zf9773ZcwZxo888kjGXHO5Pg44k5IzaZl+ZmqFDbj5PfbYY4/ifZvKVtrWGJ3XAn5Ppi/rtCz3\n8/HHH58xOyc0lUjU10fu26ZmzWxuzZm3EeVawWzWzdnSTedIfSy09brIfcayBh77vHZEdKchON93\n7733zpjpy4iyDKeeCT1cdPvYqY9dzkpmqQfLZvgd63Ni9913z5hNzLtRkuGTQ0mSJCVvDiVJkpS8\nOZQkSVJqV6FNlPl3tgpgjp0tMNjagouUR0S89rWvzZi5+x122CFj1uksXLgw44svvrh4L9YOsP6x\nqWaONVARZW0W2zy8+93vzpjtU1TifmbdGVtjcGUC7v+6nqZpZQy+L9+LxxtbUxx33HHF+7KD/XDC\nmje2romIuOOOOzKeNm1axttuu23GrHMbNWpUxqzTqs+jbbbZJuMHH3wwY44l32v77bfPeNy4cRnX\nNUeb2zJkKLBmlqvQsN1MRFm/3NSypqkGq9MWTVzVhjW3rNGNKNt2cBUdnpdNq9q0tcaw1vSd+6s9\nHsj/J88Znhccb9bcchwiyvY1bau5HUobWwfP+5TZs2cX/zZr1qyMm1aH2m677TLmdS8iYr/99uvz\nddYcSpIkqau8OZQkSVJq9bNkPupm24sbbrgh46VLl2Zcp3KZWmEKY5dddsmYj5DZpqN+BM8UAFPR\nfLzLx8v1ShL8bpdddlnGnN7e9Gi/V9IsnWpKh3D/RZTptGXLlmXMVQZ+//vfZ8w0Gz+jTiUy/cbH\n+JMmTcqYqxcwfca/Z7uOiHIlEK7E0es4LjzfIiKWL1+e8Q9/+MOMm1LMTen+OkXK/cfz+F3velfG\nXN2GaeXhliLjscxWKFzhIqK85nC1pk1pF9WE1yLu5ze/+c3F69atW5fxPvvsk/FwWh2K358rWTBd\nu3r16mIbpgZ5zDalCZuulfXnc3te717zmtdkXJ8X48eP73ObkaY+P/jnZ555JmP+HrEVzcyZMzP+\n1a9+1fg53N8sT+K5Wv9ucGW1bq9c45NDSZIkJW8OJUmSlFqdf+FjcKYCmYLiDNP6sTtXUmlapYGP\n+XfccceMmb6JKBfKfu9735sxHztzVY5zzz232H7Dhg0ZMxX6wAMPZMyVHYbbY34+qmea8g9/+EPG\nl156abEN0/xMZ3JcOeuLj+T333//jJm6jyhXeeCYMzXD79s0i7w+3pgy4/a9nj7jscgVaSLKco8b\nb7wx44suuijjE088MWPui1133TVjznqOKPfljBkzMt5rr70y5tg3rfzRH567fK+24azsiRMnZrx4\n8eLidUxT8vjj9bJp5vLmHqO77bZb8ecDDjggY6aVOU69fo7wO/MawTThj3/842IbXu94XWKnDaYP\nm1LH9Wc2rXbDY4erO0WUx3wv7v/BwuvCbbfdljHLZpYsWZIxf6fq1P3RRx+d8Ze//OWMuVoNO6Ow\nVCaiLGnqxgxl8smhJEmSkjeHkiRJSq1OKxNTI2vWrOnzNfWj8aaZdZxxynQ1m+d++MMfLt5r6tSp\nGfPxLmdfslkvFzWPKBtgPvTQQxnfd999fX5+J41rexX32YIFCzK+8MILi9cxZUmcYXzUUUdlfMwx\nx2TMRco5Uz1i41P2w23/byymuZgujIi45pprMua4zp07N2Omzzjjj6nkOnV/yCGHZNyUSt7ctGib\nU8nEaxeP63vuuad4HdP0LLHgPmdJB8+DOh3WSZqeZRVMk9XvzRKdgZw53Sa8XnMma116ct1112XM\nma1MKzO1eNhhh2XMxRsimmflcx8P1/09mFgWwM4oK1euzJjlARyHY489tnivs846K2OOBX/3/vjH\nP2bMbgwR/9tEvZt8cihJkqTkzaEkSZJSq9PKTGcwNchmkpz5W8+k5ONeplmYWmG6l+mP+hF+02xb\nzlrio2LOeKq3Hz16dMacQTac05dNY8kUSp2C4eN9ph05TiwR4ExOfkadRh7O+3kwcH9xLeuIclYy\nm4KzXIIz/Zk+43ideuqpxftyXEf6rEr+n9/0pjdlzE4HEWUZBs8dpp+ZGrv//vsznjBhQvFeTOuz\nCTlxoQCm2SLK62fTDOXhhNcYNsPnDP6IiJ/97GcZ85rG3wumnjlDtv5N4nrWBx10UMZjxozJmCUF\n9cIOLKkayepjkmledgRgORvHe++99854+vTpxXvxHJ03b17GLLthA3V2T4kY2jXHfXIoSZKk5M2h\nJEmSkjeHkiRJSq2rOWRenbUArFvhotULFy7MmLWAEWVNB2sEWHvBuhvWGtQ1g1xhhfVUixYtypjt\nHOr6ObYDYV0CV+gYKfVU3BeTJk3KmC1LIsp6D9bOsG7qnHPOyXiPPfbImC0ghttqM0Opbgt0yimn\nZMy6QbZyYssanjts/7EprVRGCu6np59+OmPuy4iy5pq1Tqyr5vVy+fLlfX5GRMQPfvCDjKdMmZIx\na0bZuqiuOeSqKFytqL4uDhc8XlkbWLc24T5jezbul9tvvz1j1tTffPPNxXtdccUVGbOO/eCDD86Y\nv6H1Cimnn356xvx9G2k4JhERc+bMyXjVqlUZc4xYJ8i//9rXvla8F9vbsYUda4J5D1CPQ7dXRSk+\ne8g+WZIkSa3jzaEkSZJS69LK1DSNm4/tubB4PSWdj2h32mmnjJkCYDqEKeabbrqpeC++N9OUjJm+\nqW299dYZs31LJ13uh1tarWksZ82aVbzuxBNPzJgLoDPlz1KASy65JONp06ZlXLdsGG77s5vq0o3r\nr78+Y6b+qakN1Lp16xq3Ha7px03B9OP8+fMz5uo0EeXqKUw/T548OePjjjsuY5Z0XHvttcV7rV27\nNuMnn3wyY7ZJYYkB02cR5Tn2lre8JWOmPznGvb4iVNNqXPvuu2/xOqboecyzXRP35bnnnpsx29pE\nRGzYsCFjrqRx1113ZczrY92SiC13ZsyYkfFQpjK7hdehut0QU8n8reG9Bstm+Po6Rc1zl6lkHvt8\nr7qcbSgN/6NAkiRJHfPmUJIkSanVaWViypazufiotk558ZEuZygzncWUCR8J12mtplnU/F58TT1D\nlqlkrj7QtHrASHi0H1HuM6a5IiK+/vWvZ/ylL30p46VLl2bM/c9UGFeFqGdBcxZ7L6awuo3n0Ve/\n+tXi3zhjkucMj2uWVDB9xll6v/vd74r3ZRnISB8vlr5wNuTDDz9cvI5dHA488MCMuerC7rvvnjFX\n2DjppJOK92IZxy233JIxr5cs22F5QUQ5M/PCCy/MeObMmRnzfO/16x2v3TwP6t8RpuK5ahePa+5X\nltp8/vOfL96L48/PueqqqzLmzFuWTUWUpVPHHHNMxjxfe/18435hqcV3v/vdjK+++upiG6Z5eYxy\nf/N3hzHLJiLK45rlArxvYbq63t9Duf97+4yUJEnSgPLmUJIkSaln0sp8vMrHvnxsXM945GPkRx99\nNGPOCGLKjOq0cFMj5aYUAh/NR0S87W1vy5jNSDtp0FzPwu71R/38/zTNAo8oZ/addtppGZ9xxhkZ\nM+XFVCZTKEyl1a+jXt+vA4nH8kc+8pGML7vssuJ1PH/YGPbwww/PeOedd86YTZQ53jw/I/73mB8M\nLENh6roN+P/nTGI2t65Tlo888kjGTE2y6TyPcf6f+ZqIsvSFs53Z0Jzb1ynuBQsWZMzr8rx58zJm\nipnftz4/eyHlzPFi+rC+pm277bYZdzJDm+NSH6PcZ/z8D33oQxmvX78+4/rcvfvuuzPmb2VTurvN\nmn5TOBYsieFxWJ9H/H3eb7/9Mub1gqUavL6NHz++eC/+Dn3yk5/MeM2aNRlzHNm9I8K0siRJklrC\nm0NJkiSlnkkr8xE80xxMQdQpYj4uZiqZDUO33377jPkIl7Mq6+05E4+f2d/2bDjalCbh9vzu9Szs\nen3btuL35oyuplmt9X5hSmXcuHEZcyYr15OdOHFin+9b77/+xqmv1zSpU5+9Ptuc3//KK6/MeO7c\nuRnX5xhTKpdffnnGY8eOzZjpTnYN4Hk0FPuLqbS2pZXpsccey7hpVmxExDbbbJPxxjaVrl/D85Xn\n0oQJEzJ+/PHHM+YYR5SlP8R0WlPpR5vHoglnny5atCjjV7/61cXrWHqxsfqbycrjgiVNvG7ydy+i\nTB/X18hew+/PriMXXXRRxrym8dit9yt/Xw499NCMeYxyYYX+zi+WEfDax/HitbLp92go9N4vmCRJ\nkgaNN4eSJElKPZlW5vqgTLmcffbZxTZMOTJ9zJl4nJnEtAzTBBHlur1N6TA+KuYj+4hyXUumTZq2\n56PqeiZhmzFVd8MNN2TMFBKb9TLtUT9SZ2qLjYAZc5Yk34spNx4HEeV+Zsqsk5nj1Csz+TrFBsef\n+MQnMua+3GWXXYptuC4pZyLzvGSj7CeeeCJjjvdhhx1WvC//bbD2c5vLM3iMMoXVX4NlrqnLFFbT\nvmSpTL0m7IoVK/qMWdLDz+PCBPXn8/rFz+S1t6mDQJtx/zOtzuOdpS4R5frXm5tCbFqznLPDubZy\nXRLSX0lP29UlPfw/c/GEiy++OGNee/orKWFzfs4+5qzipt+KOj3//e9/P2NeE6nTFHW39dYRIUmS\npEHlzaEkSZKSN4eSJElKPVPowVw86/lOOeWUjD/60Y8W27AOhHUvrK9hrQVrQNhZPaLsJs9VCjZs\n2JAxazh22223YnvWMTR1c2+qN2hTHcL/s3jx4ow/+9nPZsx6Nu4bdqCva6hYz8b6JtYtsW0Dx5gt\nVljTEdFca8P93NQmgurvyxrJXmzH8Zvf/CZj1vCwHcf5559fbMPWHDyvPve5z2X805/+NGPWmb3/\n/e/PeNq0acX7DlZLh26svDIQeFyedNJJGbOG6tZbby22Wb16dcaf+tSnMt5nn30yZt3TqlWrMub5\nGVFeY3n+3HbbbX1uX9dT7bXXXhlzzD/+8Y9n3LRaSK9oaj3GNinLli0rtuFKQHVd+v9TX29YQ7h2\n7dqM2b6FK4HUNbYzZszImDXxvfR78xz+DvM3hfvl6quvzphzCvq7VnMs2QqI+4j1hytXriy2X7hw\nYcZNq4GxrrFN+773zkhJkiQNGm8OJUmSlLboYpql6/mcpvRt02odTG3UrR2WL1+e8Ve+8pWM2VZl\n9OjRGZ955pnF9gcccEDGnbSm2cTHywPxTHqjx4n7dsmSJRkfccQRGdftZJ7Dfd5fmpaP4ZlKZguU\n2bNnZ8xUaN0mY6Ae3bMdQkT5fftJGQ3JGDXhdz7rrLMy5vE7atSojI899thie7aIuuCCCzJmGQD3\nP7f/1re+lTHbSEUMXnqlkzKOaNkYEdO/Z5xxRvFvbJvBsgC212i6JtZpXY45U/xMBTPdvOeeexbb\ns63UkUcemTFTaEzT8RjZiLHf3HHarDHi/uOKMB/84AczvuOOO4ptDj744IxPPvnkjLm/+P9nup4t\n1CIirrrqqox/8pOfZMyWLWwvdfzxxxfbn3DCCRk3tVMZgPNwUMaovm+p2/Q8h+PC1XlY6lLvV74X\ny8HGjBmTMa97/HumkSMizjvvvIxZ5sbxXrBgQcZcgSiia2nmPj/EJ4eSJElK3hxKkiQpDeu0cpNO\n/s91ipOpmTvvvDNjzkbjDFl2wo/oWhf0IU8rM9X66U9/OuP58+dnzBlgnDVW7xemmpiO4gz1o48+\nOmOmwrox+7Huhs/v38+KD61KWfI458L0XCGFqa3+zgvi/p8+fXrGP/rRjzLmigPdwu/fT/qsVWPU\npF7Fac6cORnPmjUrY66SwfOzafZkRJk243WN6eN3vOMdGdfpsLFjx2bMMhB+zgCkL4c0rVy8Efbl\nvffemzGvgRERv/71rzPmucOZxExrsjyAJSAR5b7k9WbSpEkZszxkypQpxfZdul62JvVPvA7Ur+G4\n8LeKq/7w79kloO4gsH79+ow5rqeeemrG7CYwRF0uTCtLkiSpf94cSpIkKY3ItPKmaJrtzIbMfA1n\n4kV09ti+aSy6OHsvYgAf43M/cZYlZ5CxeSn3ZUQ5U4wzwXt5wfhowRg1YQqLzYqvu+66jDkTMqJM\ng3AGHmdszpw5M+O6EW9LtXaM+sNUGdPH999/f8YrVqzImDNp67Ty1KlTM2aD5F133TXj/tKSAzzj\ntUlr0srFm+IaWDcXX7p0acYs4+BM2nXr1mXMVOR2221XvNehhx6aMTtDTJ48OeNul9r0oZVjtCk6\n6X5S/4Y1lbGwA8AmztQfSKaVJUmS1D9vDiVJkpRMKw+gDhvsDqaeTIeNMD0xRp00jY8o05FN61T3\noJ4YIw2flOUw5hi1n2llSZIk9c+bQ0mSJCXTysOL6bD2c4zazzHqDaYs288xaj/TypIkSeqfN4eS\nJElK3hxKkiQpeXMoSZKk5M2hJEmSkjeHkiRJSt4cSpIkKXlzKEmSpOTNoSRJklI3V0iRJElSy/nk\nUJIkScmbQ0mSJCVvDiVJkpS8OZQkSVLy5lCSJEnJm0NJkiQlbw4lSZKUvDmUJElS8uZQkiRJyZtD\nSZIkJW8OJUmSlLw5lCRJUvLmUJIkScmbQ0mSJCVvDiVJkpS8OZQkSVLy5lCSJEnJm0NJkiQlbw4l\nSZKUvDmUJElS8uZQkiRJyZtDSZIkJW8OJUmSlP4Dwo13QgsuqH0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f8871a207b8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAocAAAB8CAYAAAAb3hoRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGIZJREFUeJzt3Xmw1WUdx/HHUjMTMBRJxN0MMsUdiJFUGhfcitTEhsYN\nRvOPxmpyAQLBZmycctxG01GjpnItCsYFlwopHSRQVFRWMyA3UpQo0qK/+Pp+ns7vcoG7nHPu+/XX\nB+75nXvubzvP/L7PstX69euTJEmSlFJKH+nsDyBJkqT6YeNQkiRJwcahJEmSgo1DSZIkBRuHkiRJ\nCjYOJUmSFGwcSpIkKdg4lCRJUrBxKEmSpGDjUJIkScHGoSRJksLWHfi7XMS5/W3VBu/hcWpfHqP6\n5zFqDFt6nDxG7c9jVP9qHiOfHEqSJCnYOJQkSVKwcShJkqRg41CSJEmhIwektIv169fXzCmltNVW\nW9XMkiRJ9Yxtmv/85z81X/PRj340clu2c3xyKEmSpGDjUJIkScHGoSRJkkLD9Dlk7f2///1v5HXr\n1kXedttts21Yi5ekZlZ1j+T/f+QjHz4PsE+21Pl4fX7wwQfZz9i+eeWVVyIvXLgw8imnnBJ5m222\nabPP5ZNDSZIkBRuHkiRJCnVdVi6nptmAJZDtttuucntLJY2NpbF//OMfkd95552ar/nEJz4RmeUz\n/n9KKW299YenfVVpzXOndaqmkvrnP/8Z+e9//3vkHXbYITKPA3NK+f5n95CqY9fMOIXFv//978iL\nFy/OXnf//fdHZtnp9ddfj7z33ntH3mmnnSLvtttu2Xv16dMn8l577RV5xx13jMzr6pOf/GS2/cc+\n9rHIXeU4bcB70vvvv5/9jNfF9ttvX3N7Xkct3ZNas1957pTdrHiPZFb74zFeu3Zt5F/+8pfZ6x5/\n/PHIzz33XORDDjkk8kknndQeH9Enh5IkSfqQjUNJkiSFhiwrV+lq5YuuhOcCSzWrV6+OPG3atMhz\n586NPGzYsOy9jjvuuMg9evSIXFVmsdxcjaPrOJru7rvvjvzss89G5v7bd999I++6667Z++68886R\nd99998gHHHBA5Kpj1wzHqGoE4zPPPBP5O9/5TrYNz3mWNlk+XrZsWc3fx3J1Snn5k+/FGSFYYj79\n9NOz7S+//PLILD83w7HZoGr1ihUrVkSePXt2tg2P0dChQyMfdNBBkdl1gqVgHoeU8vPir3/9a+Q/\n/vGPkRctWhS5W7du2fZXXHFFZF5LzXSM6gnPF35vXXLJJZGnTp2abcOSc0vdONqDTw4lSZIUbBxK\nkiQp1HVZmaWiqkWny0kjiRNCOhqr8bC88fGPfzwyH6+//PLLkTkqk+fFr3/96+x9OXr2+OOPj8zy\nG38fz6OWSszNVtqspRx9yTLnZZddFnnevHmROXJ1wIABkdesWRP56aefzt53+fLlkd97773Iw4cP\nj/ztb387MktmzbDvWUJkifeOO+6IzH2fUn7Od+/ePfKYMWMin3zyyZFZcmTpP6V8/y9YsCAyS6Ys\nZd55553Z9vvvv3/ks88+O3JbTtLb2Xie8fuJ++7tt9/Ottljjz0i9+vXL/Iuu+wSueo+Unaz4jnC\nsjBHtc6fPz/yu+++m20/evTomtur7fAYvfrqq5F5TbLrAWflKPHaGTlyZM3/b0u2mCRJkhRsHEqS\nJCnUdVmZj9E5mo6P6vmovJzks3fv3pFZJmzpUf0GZbmaj3u5PUtmzM1UPuksVRMhc5QkR22xlLVy\n5cqar0kppbfeeivynDlzIq9atSoyR8sOGjSo5ucoy5ecSLgZSpsb8Bp58803s59dffXVkVka5n5i\n+YzrgPbt2zcyR1imlJfG2A2Ar/viF78Y+cgjj4zM67DUKMeF5ajXXnstMifFLbvKcB/85Cc/icyJ\nr7nNoYceGvmMM87I3otlUt77Hnvsscg//OEPI7NEnVJKkydPjsxR6Z///OcrP3+jqfp+4r3jC1/4\nQrYNR+Wze0z53dUa3Ib7kiOf+bk4mXlK+Xei2kY5opzfLxzR/7e//S0yzyPOBpBSfox5XnES7Pa6\npzX21SlJkqQ2ZeNQkiRJoa7LynxEy8ewt99+e+SlS5dGLstJfITPx/ucZJQjiJYsWRKZa5OmlJcp\nmVk+5uTKF154Ybb9Zz7zmZrbqHWqRgnzkTy7G7BbQPnYnSP47r333shcs5mP8DmKmRMyf+Mb38je\nt1evXpFbWvO70fA6/M1vfpP97Kmnnorcs2fPyN///vcjc4Qx9wuP3YEHHpi9L48lS6kcLc0RucT3\nbdTSJc/Z559/PjLPUa5/nFJetmIJsaWuEBuU+4n3SJa6TjvttMg8ZmeddVa2/Ysvvhj5qquuisyZ\nAxq9rMl9yRIx933Zbakt1zPme69bty7ygw8+GJldMgYOHJhtz3XOtfn4XVNOYs1Rybx2qepaSylv\nK+yzzz6RO6ILU2PeOSVJktQubBxKkiQp2DiUJElSqOs+h+xHwX5+TzzxRGQuZF6u3sBVE9gHpqpv\nGPsfllMzcDUALobNfh/s/3j33Xdn219wwQWRx44dG7lZF6VvT5yK5mc/+1lkHj+uUsD+qinlqwFw\n+poq/fv3jzxixIjIn/vc57LXNWtfUl5Xv/3tb7OfcfUOTofC/m9lP5oNOF0KV4hIKaWTTjopMq8x\n9vnkFC1Vq9iUfb4a5RrjvmE/Pd4Tt99++2ybww8/PHJbrtbD7XksP/3pT0ceNmxYtg3vn3/+858j\nr169OjLvvY1yXKrw87MPWVsqp0nhilAzZ86MzH7AnN6r7CPdbKsKdSReh+PGjYt8/fXXZ6/jvZP3\nKO57tkfeeOONbHv2yz322GMjc5q+9uKTQ0mSJAUbh5IkSQp1V1ZmGYhTMPBRPcu6fLzLUky5PafZ\nOOqooyKzTMXyR/nY9uWXX468bNmyyO+9917Nz1Uucn7LLbfU/FvGjx8fuao0pnzffve73438wAMP\nRGaZjedCOWUEy8ScDmXo0KGRTz311MhcYaWlct3mrHLQCNasWRN5wYIF2c+4egbzpk7TwfJ0Snn5\ncb/99ovMsn5rVpioWgGp3rGEyC4SnH6EKwWVP+uI+weP8ahRo7Kf/elPf4r8r3/9K3LZ9adZcH9z\nv5SlYJ6PVZl433vyySezn3G1InbL+PKXvxyZpX9eRyk17/2qvbCtwdL9bbfdFrlcWY1l4cMOOywy\np7Zjl7Wy3cBrnN9JHdGFySeHkiRJCjYOJUmSFOqurMzH83x0ysXbiY9xy1IWyywcSTl48ODIXFWF\no/24YHlK+Sg9/h6OkJ0wYULkGTNmZNuzPMAVXk455ZTIXEy7WUe+bgqWoG6++ebIHDHLkhX32dFH\nHx35zDPPzN53yJAhkbn6SdUow65e4mfJiysupJRfJ4888khkHheWf6tKabyOUspL2bxG9txzz8ib\nUxarKt/V2zHm5zz55JMjr1ixInI5wpujtzvi7+Hv4OoNKeWrp7C7QbmKVbMry8osTb722muR+f3A\n85r/X3YjOOKIIyKzfMnvN76X3ymbju0LdiebOHFiZB7TcmUszlIyevToyI8++mhkdsEot+f3WEes\nikI+OZQkSVKwcShJkqRQd2Vlqnp0ytJKSz71qU9F5mLofHTLUiIfu7MU1tJnYTmFj/Mfe+yx7HUc\nPcsFuH/84x9HvuaaayJzhGy9lbzaSznafMmSJZE52ptlKpZTRo4cGXnSpEmRywXmt3TB+66GE7aW\npSmem5ycnKV/jrJjmebZZ5+NzDJLSvkoS5b+WSZrzXXRDNdOOZJ7gxNPPDH7N+9rHVE+5+/gNZlS\nfv/kJPTslrDzzju3y+eqJ+U9bc6cOZGvvPLKyCwx9+nTJ/L3vve9yBztmlJeoq/af45I3nQ8Zuze\nMm/evMivvPJKZM6Swcn/U0rp0ksvjcx2y7Rp0yJz4utyYQWWpasWE2gvfktKkiQp2DiUJElSaJiy\nMh+psuTKR7Llo3VOEso1mDnKj2VJlhtbKp+xnMIRtRxxWZYTuA1L2fydHf3YuN7wEX5KKV199dWR\nWbLkMeNITo4gc93QtsPz8vzzz89+VjW5+7XXXht5ypQpkVk+49qwPF4p5RPVDxw4MDLXIW1mvH9w\n8mOW2FmWTSnvrsKRrVXnP/+/LD+2pusFPyNHbKaUd7dhuZuLBrQ0UX2zKGe9ePDBByM///zzkbnG\n+9tvvx35nnvuicxFGlLKu03tuuuukTnJP88DS8y1lSPK+T3E8i+PBc9XrnlcdvVg1xl2IWN7hPe+\ncmYNro3d0ddIc16RkiRJ2iw2DiVJkhRsHEqSJCnUdZ9D4tQIX//61yNPnjw5MldlSClf0PoHP/hB\nZPYN7Nu3b+RBgwZFZn+OEvsovPjii5FXrlwZuVxgnv172D+Efbi62uoBKeX7slwlg/1J+TruS077\nwL5x9jNsO9yX3/zmN7Of8ZhxMfr58+dH5rFjZh/Fso/uokWLIrPvMKe4aTZVK9Ew85qYPXt2tn2P\nHj0iDxgwIDL3M6cSqrqmys9StbIGjxmn9ijfj6s88Pdz+2ZavYN/e7nixWmnnRaZ18jChQtrbjN1\n6tTIvA5Syr9juA1XTuGqHFwlrNZn60qqVj5JKaVf/OIXkZ955pnIvA67d+8emX0U77zzzuy92K+U\n7QP2ReVUOMOHD8+278xxCD45lCRJUrBxKEmSpNAwZWWWRsaMGRP5rrvuisxHuCnlZQs+kp8+fXpk\nPlrnAthlmYuP8Fk24HQSCxYsqPz8fDzcr1+/yCxfs5TD3FVKpD179sz+zX3L489jwakhvvWtb0Xm\ndDddZf91BE6zkVJKf/nLXyJzmhqWbYjlQ5ZmOL1U+TqWXbrKseQUGLyvcDqSspTLewnvN9yX7LrC\nsnLZJYclNK5+wmk3+F5Lly7Ntudn4zQ3xxxzTOSucL8ry+VHHnlk5J///OeR2V1g1qxZkX/1q19F\nLqeyYWmS++yFF16IzOm9PvvZz2bbH3LIIZEbfSqhqhWB+P9r166NfP/990e+4447sm14LfB7h/ch\nTsPF+16Jx4j3RH4udpkrp/TqzGuhsc8ISZIktSkbh5IkSQoNU1YmlqMuvvjiyOPGjctexwXr+UiX\npTCutrJkyZLInMk/pfyxO0uWfAzMBeZLnKl+8ODBNf+fj5CbqbTSEv6dvXv3zn7GRctZ2nrkkUci\ns7sAR/ZddNFFkV0ZYMtw33O/ppSXuniNcZ9zRoCvfOUrkVlu5MwCKeVdCniNdpXrgmVa3se4X7lf\nUkppt912i8ySb9U+4z2Npa1yex5/drdh9w6W7MrPzBVeOLqd5xJXgmimY1z+LSxT8vhxpPmee+4Z\nmaNXy/sY/80VpLiy1B/+8IfIM2fOzLY/+OCDN/4HNAh2keB5ydlEbrjhhsgPPPBA5LILDK+FAw44\nIDLbB7wmeK6X1yS7rXFUMz8v2zP1NGrfJ4eSJEkKNg4lSZIUGrKszMfpZ511VmSONk4ppRkzZkTm\nCCSWRlg+5siicvQWS2B8jMzF7llaKUsAHI3Jxbn52JkliGYdvVfi31buM054PH78+Mg8TizlsxTH\n412WzLRxLHuw6wZLhOXrWDI7/fTTI7PMxRHpjz/+eORrrrkme9/99tsvMieN7yrXQv/+/SPzuuB9\nqZw0f1NHnLa0L3ksWepatWpVzdfstNNO2fbLly+PzHssu/RwtCy7Huywww4b/ezNhseO3wktnfv8\njmA589RTT41cjnBuVvwe5qwlY8eOjTx37tzIbAOUE03zeuP3CMvPbA/wnsaFNFJK6aWXXorMa4LH\nruoempKjlSVJklQnbBxKkiQpNGRZmY9aWTK84oorstfx8TJHQ7IcwxF+Vb8jpXyiS04EXDUCqcQR\nzvzMfIxcVRYqJ/hs1tJaSyP7OIJv2LBhkTmCjOUoTuLLyUtr/R79P47G48i+8hznOcvZAjjSnGWa\n1atXR3744YcjcxLglPJST1lq6Qo4epclV06+W066z3sRS8GtKTeX9xiWzXjM+L48ruXk6Oz6wRHK\nLOfxb+kq12TVZM2b8/dXzW7B38Fjx9zoyv3I74EpU6ZEZlmXo5i5fTkBPO9xfF9+b3Myc04+X07m\nz8nsqxYG6NOnT2RHK0uSJKku2TiUJElSaPh6DUsme+21V/azG2+8MfJDDz0UmSOb+EiZZcly9B0n\nXuaoZJZPqtZfTil/3FyWOTfgo27+XY1UcqlaP5IlqNaUQ8r3YpmLJaynn3468h577BG5qoRQ/k7V\nxi4Z7777buRyRPnhhx8emaVkloJZor7uuusic9LyshzzpS99qeZ7dRXcH5w4/KabborM0d4ppXTG\nGWdEHj16dOQBAwZE5uhXKkvPixYtijxnzpzIQ4cOjVw100JK+SwCPP78PQcddFDkZjrGVWtGl9ry\nPsTvod///veRq2bTaOvf39l4XnHib84swr+3pS5gLO2yS8c+++wTmZP0s6y8cuXK7L3Kf2/A853v\nW09rXNfPJ5EkSVKns3EoSZKkYONQkiRJoXk6eqT/HwbO/lDsd8NpM9ifiv0TmFPKZ/lnv4B58+ZF\nZh+5HXfcMdv++OOPr7k9NUMfkKeeeioyp6o4+uijI3NKAO6zWbNmZe91++23R2YfOE5Tw/caNWpU\n5N69e0duhv3aEdg/in08eV1xSqaUUrrkkksicz+zn9mkSZMi33fffZHZL/RrX/ta9r4HH3xwzfft\nKnheT5gwITL7jd1yyy3ZNuxrxuuQx4yrObDvczmlF88FTj8zf/78yOyj/dZbb2Xbs68W75377rtv\nZPZ/bKY+h9xf7AuYUn4tbenfzH5zPPZ33XVXZH7XnHPOOdn29dS/bVOVfTnZb48rOr3wwguRf/e7\n30V+8803I3fr1q3yvUaMGBGZ/QzZbuB1sHDhwuy9qqaqY3/d4cOHR66ne13jnh2SJElqczYOJUmS\nFJrnWX4NVQvWc0ZyThnBR9VlWZll4rJUUGubE088MfvZ2WefHZmPoaumrKmnx8sp5fum/Gwso3Al\nGi563r1798gsc/ExfLnKAn8nt2HJasyYMZFZNmHJrN72Zb3ifmLJj/u+LEU9+eSTkbkCw7333huZ\nXS94rgwePDjyhRdemL0vVzHq6lhi/tGPfhT5hBNOyF537rnnRmbXC07XxalNeLxZfkspX6GG5s6d\nG5n3sd133z17HY/zkCFDIg8aNCgyu340cokzpbx7DKfxWbVqVfY6TrfGcmbV9wDvgeX3zq233hp5\n4sSJNbf/6U9/Grlfv34t/g2NpLyn81y86KKLIrP0zszzs8QuMez6wi4BfC8eu3K1Fa4Cxfsor4nD\nDjus8rN0psa+IiVJktSmbBxKkiQpbNXSDO5trMN+0ca0Zgb7cgZ1lg04+m7FihWR+di4V69e2fYs\nq7Lc3VK5dmP/X+ulrX1hCzZ6nMp9xkfpixcvjsxVGrjiAkdcclRsWVpiKZ8lY67EwdGX5eoddapD\njtGWYmmFK3Q88cQT2et4XbAUyRIY///YY4+NzJJXjx49tvATt6mGOEbldchuLdOmTYvMUf8cxczX\nl2U2lvU5spKZozdZLk4ppf79+0c+5phjInO0Mt+rqoy9EVt6nLboGFWN7n/uuecilzMwVHWr4AhZ\nXntcmWvKlCnZe7300kuR2VXq5ptvjnzUUUdF7qTSfaceoy3VmvYR2wo8dimlNHPmzMjsusFuZ7xW\nyhlXOkjNY+STQ0mSJAUbh5IkSQpdsqy8Oar2E8sEzOXj4daUhjkyiiWATZgstVPKYdw3HLH66quv\nRp4xY0ZkjuZj6Zglp5TyyUFbM7KvQTREyZKlEo4o50TjKaW0dOnSyDz/WSbmqHWOqO2kEkprNMQx\navGX45rkfWXNmjWRWeaaPn16tj27wRC7cbAsyhkEyu15vfK8qjr+HdiNps3KyuxGsWzZssi33XZb\ntg0nX+a9ksdl+fLlkXlNlV2VvvrVr0Y+77zzIvNeWQf3x4YuK2+qsp1QNSE6v9N5HXTS8bKsLEmS\npJbZOJQkSVKwrNyGWDIpHw+35nExH0FvZum008vKdVDGqHcNV7Lk8eVI85Ty0Xk8ZzkpLUe+Nsj5\n0XDHqIuqm5Il7/0cBV5Ogs21qTnRPEuOHAXOXK5rzhHedTyJeN0cI1WyrCxJkqSW2TiUJElSsKzc\nhra0vFp1LOq9rKxN4jGqfx6jxmDJsv55jOqfZWVJkiS1zMahJEmSgo1DSZIkhVYvvaFNU/YfbE2/\nwQaZ5kOSJDUxnxxKkiQp2DiUJElSsKzchiwLS5KkRueTQ0mSJAUbh5IkSQoduUKKJEmS6pxPDiVJ\nkhRsHEqSJCnYOJQkSVKwcShJkqRg41CSJEnBxqEkSZKCjUNJkiQFG4eSJEkKNg4lSZIUbBxKkiQp\n2DiUJElSsHEoSZKkYONQkiRJwcahJEmSgo1DSZIkBRuHkiRJCjYOJUmSFGwcSpIkKdg4lCRJUrBx\nKEmSpGDjUJIkScHGoSRJkoKNQ0mSJIX/ATb2biziZCxeAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f88332186a0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "n_iterations = 3\n",
    "n_digits = 6\n",
    "codings_rnd = np.random.normal(size=[n_digits, n_hidden3])\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_variational.ckpt\")\n",
    "    target_codings = np.roll(codings_rnd, -1, axis=0)\n",
    "    for iteration in range(n_iterations + 1):\n",
    "        codings_interpolate = codings_rnd + (target_codings - codings_rnd) * iteration / n_iterations\n",
    "        outputs_val = outputs.eval(feed_dict={codings: codings_interpolate})\n",
    "        plt.figure(figsize=(11, 1.5*n_iterations))\n",
    "        for digit_index in range(n_digits):\n",
    "            plt.subplot(1, n_digits, digit_index + 1)\n",
    "            plot_image(outputs_val[digit_index])\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Exercise solutions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Coming soon..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  },
  "nav_menu": {
   "height": "381px",
   "width": "453px"
  },
  "toc": {
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 6,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
